2024-11-10 17:22:01,087 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 17:22:01,103 main DEBUG Took 0.012913 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 17:22:01,103 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 17:22:01,104 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 17:22:01,105 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 17:22:01,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,116 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 17:22:01,132 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,135 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,136 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,136 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,138 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,139 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,141 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,142 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,143 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,143 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,144 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,145 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,146 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 17:22:01,147 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,148 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 17:22:01,150 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 17:22:01,152 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 17:22:01,154 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 17:22:01,155 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 17:22:01,156 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 17:22:01,157 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 17:22:01,168 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 17:22:01,171 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 17:22:01,173 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 17:22:01,173 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 17:22:01,174 main DEBUG createAppenders(={Console}) 2024-11-10 17:22:01,174 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 17:22:01,175 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 17:22:01,175 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 17:22:01,176 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 17:22:01,176 main DEBUG OutputStream closed 2024-11-10 17:22:01,176 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 17:22:01,177 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 17:22:01,177 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 17:22:01,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 17:22:01,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 17:22:01,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 17:22:01,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 17:22:01,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 17:22:01,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 17:22:01,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 17:22:01,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 17:22:01,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 17:22:01,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 17:22:01,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 17:22:01,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 17:22:01,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 17:22:01,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 17:22:01,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 17:22:01,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 17:22:01,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 17:22:01,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 17:22:01,289 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 17:22:01,290 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 17:22:01,290 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 17:22:01,291 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T17:22:01,603 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce 2024-11-10 17:22:01,606 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 17:22:01,607 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T17:22:01,616 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-10T17:22:01,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T17:22:01,658 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9, deleteOnExit=true 2024-11-10T17:22:01,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T17:22:01,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/test.cache.data in system properties and HBase conf 2024-11-10T17:22:01,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T17:22:01,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/hadoop.log.dir in system properties and HBase conf 2024-11-10T17:22:01,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T17:22:01,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T17:22:01,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T17:22:01,788 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T17:22:01,927 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T17:22:01,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T17:22:01,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T17:22:01,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T17:22:01,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T17:22:01,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T17:22:01,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T17:22:01,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T17:22:01,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T17:22:01,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T17:22:01,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/nfs.dump.dir in system properties and HBase conf 2024-11-10T17:22:01,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/java.io.tmpdir in system properties and HBase conf 2024-11-10T17:22:01,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T17:22:01,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T17:22:01,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T17:22:02,927 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T17:22:03,031 INFO [Time-limited test {}] log.Log(170): Logging initialized @2804ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T17:22:03,127 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T17:22:03,195 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T17:22:03,218 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T17:22:03,218 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T17:22:03,219 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T17:22:03,232 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T17:22:03,235 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/hadoop.log.dir/,AVAILABLE} 2024-11-10T17:22:03,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T17:22:03,471 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/java.io.tmpdir/jetty-localhost-37613-hadoop-hdfs-3_4_1-tests_jar-_-any-7177619379186237109/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T17:22:03,488 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37613} 2024-11-10T17:22:03,489 INFO [Time-limited test {}] server.Server(415): Started @3263ms 2024-11-10T17:22:03,977 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T17:22:03,987 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T17:22:03,989 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T17:22:03,989 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T17:22:03,989 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T17:22:03,990 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/hadoop.log.dir/,AVAILABLE} 2024-11-10T17:22:03,991 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T17:22:04,128 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/java.io.tmpdir/jetty-localhost-33175-hadoop-hdfs-3_4_1-tests_jar-_-any-11977271902139280881/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T17:22:04,129 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:33175} 2024-11-10T17:22:04,129 INFO [Time-limited test {}] server.Server(415): Started @3903ms 2024-11-10T17:22:04,189 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T17:22:04,707 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/data/data1/current/BP-838508068-172.17.0.2-1731259322633/current, will proceed with Du for space computation calculation, 2024-11-10T17:22:04,707 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/data/data2/current/BP-838508068-172.17.0.2-1731259322633/current, will proceed with Du for space computation calculation, 2024-11-10T17:22:04,745 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T17:22:04,804 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8770e9b25aadf40 with lease ID 0x390285ac3a8523c0: Processing first storage report for DS-756b1a0d-d7ea-4796-a973-050d21e94f72 from datanode DatanodeRegistration(127.0.0.1:36153, datanodeUuid=4ee7403d-08d9-4cb1-a51e-17e70bcbf226, infoPort=42633, infoSecurePort=0, ipcPort=41805, storageInfo=lv=-57;cid=testClusterID;nsid=6228564;c=1731259322633) 2024-11-10T17:22:04,806 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8770e9b25aadf40 with lease ID 0x390285ac3a8523c0: from storage DS-756b1a0d-d7ea-4796-a973-050d21e94f72 node DatanodeRegistration(127.0.0.1:36153, datanodeUuid=4ee7403d-08d9-4cb1-a51e-17e70bcbf226, infoPort=42633, infoSecurePort=0, ipcPort=41805, storageInfo=lv=-57;cid=testClusterID;nsid=6228564;c=1731259322633), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T17:22:04,806 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8770e9b25aadf40 with lease ID 0x390285ac3a8523c0: Processing first storage report for DS-6db00a98-dd03-4b95-957b-a1c90ce32256 from datanode DatanodeRegistration(127.0.0.1:36153, datanodeUuid=4ee7403d-08d9-4cb1-a51e-17e70bcbf226, infoPort=42633, infoSecurePort=0, ipcPort=41805, storageInfo=lv=-57;cid=testClusterID;nsid=6228564;c=1731259322633) 2024-11-10T17:22:04,807 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8770e9b25aadf40 with lease ID 0x390285ac3a8523c0: from storage DS-6db00a98-dd03-4b95-957b-a1c90ce32256 node DatanodeRegistration(127.0.0.1:36153, datanodeUuid=4ee7403d-08d9-4cb1-a51e-17e70bcbf226, infoPort=42633, infoSecurePort=0, ipcPort=41805, storageInfo=lv=-57;cid=testClusterID;nsid=6228564;c=1731259322633), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T17:22:04,875 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce 2024-11-10T17:22:04,968 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/zookeeper_0, clientPort=49964, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T17:22:04,979 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49964 2024-11-10T17:22:04,995 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T17:22:04,999 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T17:22:05,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741825_1001 (size=7) 2024-11-10T17:22:05,671 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a with version=8 2024-11-10T17:22:05,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/hbase-staging 2024-11-10T17:22:05,779 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T17:22:06,026 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8e3de9160f9f:0 server-side Connection retries=45 2024-11-10T17:22:06,038 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T17:22:06,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T17:22:06,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T17:22:06,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T17:22:06,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T17:22:06,192 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T17:22:06,252 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T17:22:06,262 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T17:22:06,266 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T17:22:06,299 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28167 (auto-detected) 2024-11-10T17:22:06,300 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T17:22:06,319 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40007 2024-11-10T17:22:06,341 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40007 connecting to ZooKeeper ensemble=127.0.0.1:49964 2024-11-10T17:22:06,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400070x0, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T17:22:06,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40007-0x10102e6c4a10000 connected 2024-11-10T17:22:06,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T17:22:06,408 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T17:22:06,421 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T17:22:06,426 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a, hbase.cluster.distributed=false 2024-11-10T17:22:06,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T17:22:06,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40007 2024-11-10T17:22:06,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40007 2024-11-10T17:22:06,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40007 2024-11-10T17:22:06,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40007 2024-11-10T17:22:06,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40007 2024-11-10T17:22:06,595 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8e3de9160f9f:0 server-side Connection retries=45 2024-11-10T17:22:06,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T17:22:06,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T17:22:06,598 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T17:22:06,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T17:22:06,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T17:22:06,601 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T17:22:06,604 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T17:22:06,605 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36421 2024-11-10T17:22:06,607 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36421 connecting to ZooKeeper ensemble=127.0.0.1:49964 2024-11-10T17:22:06,609 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T17:22:06,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T17:22:06,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364210x0, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T17:22:06,621 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:364210x0, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T17:22:06,622 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36421-0x10102e6c4a10001 connected 2024-11-10T17:22:06,626 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T17:22:06,636 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T17:22:06,639 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T17:22:06,644 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T17:22:06,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36421 2024-11-10T17:22:06,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36421 2024-11-10T17:22:06,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36421 2024-11-10T17:22:06,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36421 2024-11-10T17:22:06,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36421 2024-11-10T17:22:06,673 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8e3de9160f9f:40007 2024-11-10T17:22:06,674 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:06,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T17:22:06,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T17:22:06,683 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:06,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T17:22:06,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:06,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:06,705 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T17:22:06,706 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8e3de9160f9f,40007,1731259325831 from backup master directory 2024-11-10T17:22:06,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:06,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T17:22:06,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T17:22:06,710 WARN [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T17:22:06,710 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:06,712 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T17:22:06,713 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T17:22:06,772 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/hbase.id] with ID: b79f36ae-4c52-42ea-8782-f05a267fe934 2024-11-10T17:22:06,772 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/.tmp/hbase.id 2024-11-10T17:22:06,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741826_1002 (size=42) 2024-11-10T17:22:07,184 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/.tmp/hbase.id]:[hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/hbase.id] 2024-11-10T17:22:07,232 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T17:22:07,237 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T17:22:07,258 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-10T17:22:07,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741827_1003 (size=196) 2024-11-10T17:22:07,299 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:22:07,302 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T17:22:07,320 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:07,325 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T17:22:07,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741828_1004 (size=1189) 2024-11-10T17:22:07,380 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store 2024-11-10T17:22:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741829_1005 (size=34) 2024-11-10T17:22:07,403 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T17:22:07,406 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:07,407 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T17:22:07,407 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:22:07,408 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:22:07,409 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T17:22:07,409 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:22:07,410 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:22:07,411 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731259327407Disabling compacts and flushes for region at 1731259327407Disabling writes for close at 1731259327409 (+2 ms)Writing region close event to WAL at 1731259327410 (+1 ms)Closed at 1731259327410 2024-11-10T17:22:07,413 WARN [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/.initializing 2024-11-10T17:22:07,413 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/WALs/8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:07,424 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T17:22:07,441 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8e3de9160f9f%2C40007%2C1731259325831, suffix=, logDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/WALs/8e3de9160f9f,40007,1731259325831, archiveDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/oldWALs, maxLogs=10 2024-11-10T17:22:07,466 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/WALs/8e3de9160f9f,40007,1731259325831/8e3de9160f9f%2C40007%2C1731259325831.1731259327447, exclude list is [], retry=0 2024-11-10T17:22:07,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36153,DS-756b1a0d-d7ea-4796-a973-050d21e94f72,DISK] 2024-11-10T17:22:07,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T17:22:07,533 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/WALs/8e3de9160f9f,40007,1731259325831/8e3de9160f9f%2C40007%2C1731259325831.1731259327447 2024-11-10T17:22:07,535 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42633:42633)] 2024-11-10T17:22:07,535 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:22:07,536 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:07,541 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,542 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T17:22:07,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:07,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:07,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T17:22:07,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:07,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:07,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T17:22:07,643 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:07,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:07,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T17:22:07,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:07,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:07,650 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,654 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,656 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,663 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,664 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,668 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T17:22:07,671 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T17:22:07,677 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:22:07,678 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75030440, jitterRate=0.11804068088531494}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T17:22:07,684 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731259327555Initializing all the Stores at 1731259327557 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731259327558 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259327558Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259327559 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259327559Cleaning up temporary data from old regions at 1731259327664 (+105 ms)Region opened successfully at 1731259327683 (+19 ms) 2024-11-10T17:22:07,686 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T17:22:07,727 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15fbfacd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8e3de9160f9f/172.17.0.2:0 2024-11-10T17:22:07,764 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T17:22:07,777 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T17:22:07,778 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T17:22:07,781 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T17:22:07,782 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T17:22:07,788 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-10T17:22:07,788 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T17:22:07,819 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T17:22:07,829 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T17:22:07,831 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T17:22:07,834 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T17:22:07,836 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T17:22:07,839 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T17:22:07,843 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T17:22:07,847 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T17:22:07,849 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T17:22:07,851 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T17:22:07,853 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T17:22:07,875 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T17:22:07,877 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T17:22:07,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T17:22:07,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T17:22:07,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,884 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8e3de9160f9f,40007,1731259325831, sessionid=0x10102e6c4a10000, setting cluster-up flag (Was=false) 2024-11-10T17:22:07,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,905 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T17:22:07,906 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:07,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:07,919 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T17:22:07,921 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:07,927 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T17:22:07,956 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(746): ClusterId : b79f36ae-4c52-42ea-8782-f05a267fe934 2024-11-10T17:22:07,960 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T17:22:07,965 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T17:22:07,965 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T17:22:07,971 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T17:22:07,972 DEBUG [RS:0;8e3de9160f9f:36421 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253d524a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8e3de9160f9f/172.17.0.2:0 2024-11-10T17:22:07,987 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8e3de9160f9f:36421 2024-11-10T17:22:07,991 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T17:22:07,991 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T17:22:07,991 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T17:22:07,994 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(2659): reportForDuty to master=8e3de9160f9f,40007,1731259325831 with port=36421, startcode=1731259326553 2024-11-10T17:22:08,007 DEBUG [RS:0;8e3de9160f9f:36421 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T17:22:08,019 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T17:22:08,032 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T17:22:08,044 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T17:22:08,052 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36701, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T17:22:08,052 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8e3de9160f9f,40007,1731259325831 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T17:22:08,060 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T17:22:08,060 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40007 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:08,060 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T17:22:08,061 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T17:22:08,061 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T17:22:08,061 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8e3de9160f9f:0, corePoolSize=10, maxPoolSize=10 2024-11-10T17:22:08,061 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,062 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T17:22:08,062 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,065 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731259358065 2024-11-10T17:22:08,066 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T17:22:08,067 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T17:22:08,068 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T17:22:08,069 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T17:22:08,071 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T17:22:08,072 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T17:22:08,072 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T17:22:08,072 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T17:22:08,076 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:08,076 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T17:22:08,076 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,080 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T17:22:08,081 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T17:22:08,082 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T17:22:08,088 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T17:22:08,089 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T17:22:08,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741831_1007 (size=1321) 2024-11-10T17:22:08,093 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-10T17:22:08,093 WARN [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-10T17:22:08,095 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.large.0-1731259328090,5,FailOnTimeoutGroup] 2024-11-10T17:22:08,096 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.small.0-1731259328096,5,FailOnTimeoutGroup] 2024-11-10T17:22:08,096 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,096 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T17:22:08,098 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,098 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,195 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(2659): reportForDuty to master=8e3de9160f9f,40007,1731259325831 with port=36421, startcode=1731259326553 2024-11-10T17:22:08,198 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40007 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:08,201 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40007 {}] master.ServerManager(517): Registering regionserver=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:08,210 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:22:08,210 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37703 2024-11-10T17:22:08,211 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T17:22:08,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T17:22:08,217 DEBUG [RS:0;8e3de9160f9f:36421 {}] zookeeper.ZKUtil(111): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:08,217 WARN [RS:0;8e3de9160f9f:36421 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T17:22:08,217 INFO [RS:0;8e3de9160f9f:36421 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T17:22:08,217 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/WALs/8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:08,219 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8e3de9160f9f,36421,1731259326553] 2024-11-10T17:22:08,247 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T17:22:08,261 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T17:22:08,267 INFO [RS:0;8e3de9160f9f:36421 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T17:22:08,267 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,269 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T17:22:08,275 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T17:22:08,277 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,277 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,278 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,278 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,278 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,278 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,279 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8e3de9160f9f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T17:22:08,279 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,279 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,279 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,280 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,280 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,280 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T17:22:08,280 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8e3de9160f9f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T17:22:08,281 DEBUG [RS:0;8e3de9160f9f:36421 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T17:22:08,284 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,284 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,285 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,285 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,285 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,285 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,36421,1731259326553-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T17:22:08,308 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T17:22:08,310 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,36421,1731259326553-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,311 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,311 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.Replication(171): 8e3de9160f9f,36421,1731259326553 started 2024-11-10T17:22:08,331 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:08,331 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1482): Serving as 8e3de9160f9f,36421,1731259326553, RpcServer on 8e3de9160f9f/172.17.0.2:36421, sessionid=0x10102e6c4a10001 2024-11-10T17:22:08,332 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T17:22:08,332 DEBUG [RS:0;8e3de9160f9f:36421 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:08,332 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8e3de9160f9f,36421,1731259326553' 2024-11-10T17:22:08,333 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T17:22:08,334 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T17:22:08,334 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T17:22:08,335 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T17:22:08,335 DEBUG [RS:0;8e3de9160f9f:36421 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:08,335 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8e3de9160f9f,36421,1731259326553' 2024-11-10T17:22:08,335 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T17:22:08,336 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T17:22:08,336 DEBUG [RS:0;8e3de9160f9f:36421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T17:22:08,337 INFO [RS:0;8e3de9160f9f:36421 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T17:22:08,337 INFO [RS:0;8e3de9160f9f:36421 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T17:22:08,442 INFO [RS:0;8e3de9160f9f:36421 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T17:22:08,447 INFO [RS:0;8e3de9160f9f:36421 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8e3de9160f9f%2C36421%2C1731259326553, suffix=, logDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/WALs/8e3de9160f9f,36421,1731259326553, archiveDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/oldWALs, maxLogs=32 2024-11-10T17:22:08,465 DEBUG [RS:0;8e3de9160f9f:36421 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/WALs/8e3de9160f9f,36421,1731259326553/8e3de9160f9f%2C36421%2C1731259326553.1731259328449, exclude list is [], retry=0 2024-11-10T17:22:08,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36153,DS-756b1a0d-d7ea-4796-a973-050d21e94f72,DISK] 2024-11-10T17:22:08,474 INFO [RS:0;8e3de9160f9f:36421 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/WALs/8e3de9160f9f,36421,1731259326553/8e3de9160f9f%2C36421%2C1731259326553.1731259328449 2024-11-10T17:22:08,475 DEBUG [RS:0;8e3de9160f9f:36421 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42633:42633)] 2024-11-10T17:22:08,495 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T17:22:08,496 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:22:08,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741833_1009 (size=32) 2024-11-10T17:22:08,908 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:08,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T17:22:08,914 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T17:22:08,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:08,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:08,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T17:22:08,918 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T17:22:08,918 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:08,919 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:08,919 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T17:22:08,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T17:22:08,922 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:08,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:08,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T17:22:08,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T17:22:08,925 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:08,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:08,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T17:22:08,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740 2024-11-10T17:22:08,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740 2024-11-10T17:22:08,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T17:22:08,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T17:22:08,934 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:22:08,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T17:22:08,940 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:22:08,942 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66352716, jitterRate=-0.011267483234405518}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:22:08,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731259328908Initializing all the Stores at 1731259328911 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731259328911Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731259328911Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259328911Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731259328911Cleaning up temporary data from old regions at 1731259328933 (+22 ms)Region opened successfully at 1731259328944 (+11 ms) 2024-11-10T17:22:08,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T17:22:08,944 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T17:22:08,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T17:22:08,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T17:22:08,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T17:22:08,946 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T17:22:08,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731259328944Disabling compacts and flushes for region at 1731259328944Disabling writes for close at 1731259328945 (+1 ms)Writing region close event to WAL at 1731259328946 (+1 ms)Closed at 1731259328946 2024-11-10T17:22:08,949 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T17:22:08,949 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T17:22:08,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T17:22:08,968 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T17:22:08,972 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T17:22:09,125 DEBUG [8e3de9160f9f:40007 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T17:22:09,138 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:09,144 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8e3de9160f9f,36421,1731259326553, state=OPENING 2024-11-10T17:22:09,150 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T17:22:09,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:09,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:22:09,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T17:22:09,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T17:22:09,155 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T17:22:09,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:22:09,333 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T17:22:09,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46989, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T17:22:09,348 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T17:22:09,348 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T17:22:09,349 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T17:22:09,352 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8e3de9160f9f%2C36421%2C1731259326553.meta, suffix=.meta, logDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/WALs/8e3de9160f9f,36421,1731259326553, archiveDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/oldWALs, maxLogs=32 2024-11-10T17:22:09,370 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/WALs/8e3de9160f9f,36421,1731259326553/8e3de9160f9f%2C36421%2C1731259326553.meta.1731259329354.meta, exclude list is [], retry=0 2024-11-10T17:22:09,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36153,DS-756b1a0d-d7ea-4796-a973-050d21e94f72,DISK] 2024-11-10T17:22:09,377 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/WALs/8e3de9160f9f,36421,1731259326553/8e3de9160f9f%2C36421%2C1731259326553.meta.1731259329354.meta 2024-11-10T17:22:09,377 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42633:42633)] 2024-11-10T17:22:09,378 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:22:09,379 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T17:22:09,382 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T17:22:09,386 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T17:22:09,391 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T17:22:09,391 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:09,392 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T17:22:09,392 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T17:22:09,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T17:22:09,396 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T17:22:09,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:09,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:09,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T17:22:09,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T17:22:09,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:09,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:09,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T17:22:09,401 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T17:22:09,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:09,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:09,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T17:22:09,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T17:22:09,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:09,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T17:22:09,404 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T17:22:09,406 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740 2024-11-10T17:22:09,408 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740 2024-11-10T17:22:09,410 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T17:22:09,410 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T17:22:09,411 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:22:09,413 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T17:22:09,415 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71197915, jitterRate=0.06093160808086395}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:22:09,415 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T17:22:09,417 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731259329392Writing region info on filesystem at 1731259329392Initializing all the Stores at 1731259329394 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731259329394Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731259329395 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259329395Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731259329395Cleaning up temporary data from old regions at 1731259329410 (+15 ms)Running coprocessor post-open hooks at 1731259329415 (+5 ms)Region opened successfully at 1731259329416 (+1 ms) 2024-11-10T17:22:09,424 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731259329324 2024-11-10T17:22:09,436 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T17:22:09,436 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T17:22:09,438 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:09,439 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8e3de9160f9f,36421,1731259326553, state=OPEN 2024-11-10T17:22:09,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T17:22:09,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T17:22:09,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T17:22:09,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T17:22:09,445 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:09,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T17:22:09,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8e3de9160f9f,36421,1731259326553 in 289 msec 2024-11-10T17:22:09,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T17:22:09,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 495 msec 2024-11-10T17:22:09,458 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T17:22:09,458 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T17:22:09,479 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:09,480 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:09,501 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:09,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41173, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:09,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5660 sec 2024-11-10T17:22:09,528 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731259329528, completionTime=-1 2024-11-10T17:22:09,532 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T17:22:09,532 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T17:22:09,561 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T17:22:09,561 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731259389561 2024-11-10T17:22:09,561 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731259449561 2024-11-10T17:22:09,561 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-10T17:22:09,563 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,40007,1731259325831-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:09,563 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,40007,1731259325831-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:09,564 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,40007,1731259325831-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:09,565 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8e3de9160f9f:40007, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:09,565 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:09,566 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:09,572 DEBUG [master/8e3de9160f9f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T17:22:09,593 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.883sec 2024-11-10T17:22:09,594 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T17:22:09,595 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T17:22:09,596 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T17:22:09,597 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T17:22:09,597 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T17:22:09,598 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,40007,1731259325831-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T17:22:09,598 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,40007,1731259325831-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T17:22:09,606 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T17:22:09,607 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T17:22:09,607 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,40007,1731259325831-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T17:22:09,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:09,670 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T17:22:09,670 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T17:22:09,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:09,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:09,684 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e667ddd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:09,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:09,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:09,698 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:09,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:09,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:09,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=6903 2024-11-10T17:22:09,750 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T17:22:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b74020b 2024-11-10T17:22:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T17:22:09,778 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T17:22:09,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:22:09,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:22:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:09,799 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T17:22:09,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T17:22:09,802 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:09,804 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T17:22:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T17:22:09,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741835_1011 (size=959) 2024-11-10T17:22:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T17:22:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T17:22:10,224 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:22:10,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741836_1012 (size=53) 2024-11-10T17:22:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T17:22:10,637 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:10,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 86036b2e5b1a6747cefd43fa6b653d49, disabling compactions & flushes 2024-11-10T17:22:10,638 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:10,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:10,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. after waiting 0 ms 2024-11-10T17:22:10,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:10,638 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:10,638 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86036b2e5b1a6747cefd43fa6b653d49: Waiting for close lock at 1731259330637Disabling compacts and flushes for region at 1731259330637Disabling writes for close at 1731259330638 (+1 ms)Writing region close event to WAL at 1731259330638Closed at 1731259330638 2024-11-10T17:22:10,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T17:22:10,648 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731259330642"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731259330642"}]},"ts":"1731259330642"} 2024-11-10T17:22:10,654 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T17:22:10,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T17:22:10,662 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259330657"}]},"ts":"1731259330657"} 2024-11-10T17:22:10,669 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T17:22:10,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, ASSIGN}] 2024-11-10T17:22:10,675 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, ASSIGN 2024-11-10T17:22:10,678 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, ASSIGN; state=OFFLINE, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=false 2024-11-10T17:22:10,831 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86036b2e5b1a6747cefd43fa6b653d49, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:10,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, ASSIGN because future has completed 2024-11-10T17:22:10,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:22:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T17:22:11,003 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:11,003 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:22:11,004 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,004 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:11,004 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,005 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,008 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,011 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86036b2e5b1a6747cefd43fa6b653d49 columnFamilyName A 2024-11-10T17:22:11,011 DEBUG [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:11,012 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] regionserver.HStore(327): Store=86036b2e5b1a6747cefd43fa6b653d49/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:11,013 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,016 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86036b2e5b1a6747cefd43fa6b653d49 columnFamilyName B 2024-11-10T17:22:11,016 DEBUG [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:11,017 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] regionserver.HStore(327): Store=86036b2e5b1a6747cefd43fa6b653d49/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:11,017 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,025 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86036b2e5b1a6747cefd43fa6b653d49 columnFamilyName C 2024-11-10T17:22:11,025 DEBUG [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:11,026 INFO [StoreOpener-86036b2e5b1a6747cefd43fa6b653d49-1 {}] regionserver.HStore(327): Store=86036b2e5b1a6747cefd43fa6b653d49/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:11,026 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,031 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,035 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,041 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,041 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,042 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:22:11,044 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,051 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:22:11,052 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 86036b2e5b1a6747cefd43fa6b653d49; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59927114, jitterRate=-0.1070164144039154}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:22:11,053 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:11,054 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 86036b2e5b1a6747cefd43fa6b653d49: Running coprocessor pre-open hook at 1731259331005Writing region info on filesystem at 1731259331005Initializing all the Stores at 1731259331007 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259331007Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259331007Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259331007Cleaning up temporary data from old regions at 1731259331041 (+34 ms)Running coprocessor post-open hooks at 1731259331053 (+12 ms)Region opened successfully at 1731259331054 (+1 ms) 2024-11-10T17:22:11,057 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., pid=6, masterSystemTime=1731259330993 2024-11-10T17:22:11,062 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:11,063 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:11,064 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86036b2e5b1a6747cefd43fa6b653d49, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:11,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:22:11,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T17:22:11,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 in 235 msec 2024-11-10T17:22:11,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T17:22:11,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, ASSIGN in 409 msec 2024-11-10T17:22:11,087 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T17:22:11,087 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259331087"}]},"ts":"1731259331087"} 2024-11-10T17:22:11,097 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T17:22:11,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T17:22:11,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3100 sec 2024-11-10T17:22:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T17:22:11,964 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:11,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:11,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:11,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:11,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ac3455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:11,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:11,978 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549869ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:11,981 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:11,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:11,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:11,996 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T17:22:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10a3757 2024-11-10T17:22:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T17:22:12,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T17:22:12,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,015 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60f8ac21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6dc539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,030 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654d6699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,039 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688e5843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,050 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c373c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4834e559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,063 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44447930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,075 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@537ae38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,081 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723bb0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,084 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235bf972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,088 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5005e0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,091 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d70ac03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5180f463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,101 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bfd911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,105 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3d4946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,107 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d424882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:12,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:12,120 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4028ac11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:12,123 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad87867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:12,135 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,136 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,142 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,143 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,154 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,171 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,202 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T17:22:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T17:22:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,216 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:12,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:12,219 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,226 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:12,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:22:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:12,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T17:22:12,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/bc18cf810cf54772a7193496816d258c is 50, key is test_row_0/A:col10/1731259332218/Put/seqid=0 2024-11-10T17:22:12,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:12,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:12,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741837_1013 (size=12001) 2024-11-10T17:22:12,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/bc18cf810cf54772a7193496816d258c 2024-11-10T17:22:12,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259392489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259392489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259392494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259392501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259392502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T17:22:12,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:12,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/2b7008fb89674c3dab5e982a7dcde82e is 50, key is test_row_0/B:col10/1731259332218/Put/seqid=0 2024-11-10T17:22:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259392646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259392647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259392648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741838_1014 (size=12001) 2024-11-10T17:22:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/2b7008fb89674c3dab5e982a7dcde82e 2024-11-10T17:22:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259392657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259392658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/719fc7ee3ab4450a8d36365539d8fe64 is 50, key is test_row_0/C:col10/1731259332218/Put/seqid=0 2024-11-10T17:22:12,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741839_1015 (size=12001) 2024-11-10T17:22:12,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/719fc7ee3ab4450a8d36365539d8fe64 2024-11-10T17:22:12,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:12,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/bc18cf810cf54772a7193496816d258c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/bc18cf810cf54772a7193496816d258c 2024-11-10T17:22:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/bc18cf810cf54772a7193496816d258c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T17:22:12,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/2b7008fb89674c3dab5e982a7dcde82e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/2b7008fb89674c3dab5e982a7dcde82e 2024-11-10T17:22:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T17:22:12,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/2b7008fb89674c3dab5e982a7dcde82e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T17:22:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/719fc7ee3ab4450a8d36365539d8fe64 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/719fc7ee3ab4450a8d36365539d8fe64 2024-11-10T17:22:12,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/719fc7ee3ab4450a8d36365539d8fe64, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T17:22:12,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 86036b2e5b1a6747cefd43fa6b653d49 in 616ms, sequenceid=12, compaction requested=false 2024-11-10T17:22:12,892 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T17:22:12,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:12,912 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:12,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T17:22:12,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/63ab4a29ea3945c59552012c1e332568 is 50, key is test_row_0/A:col10/1731259332492/Put/seqid=0 2024-11-10T17:22:12,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259392922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259392928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259392935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:12,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:12,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259392957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:12,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259392958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741840_1016 (size=16681) 2024-11-10T17:22:13,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/63ab4a29ea3945c59552012c1e332568 2024-11-10T17:22:13,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7047d3abf1e946bfbb6a68770d1e7b07 is 50, key is test_row_0/B:col10/1731259332492/Put/seqid=0 2024-11-10T17:22:13,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259393064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259393069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259393064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259393092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259393112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741841_1017 (size=12001) 2024-11-10T17:22:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7047d3abf1e946bfbb6a68770d1e7b07 2024-11-10T17:22:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/48f876ea596144f09741acc90b907c87 is 50, key is test_row_0/C:col10/1731259332492/Put/seqid=0 2024-11-10T17:22:13,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741842_1018 (size=12001) 2024-11-10T17:22:13,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/48f876ea596144f09741acc90b907c87 2024-11-10T17:22:13,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/63ab4a29ea3945c59552012c1e332568 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/63ab4a29ea3945c59552012c1e332568 2024-11-10T17:22:13,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/63ab4a29ea3945c59552012c1e332568, entries=250, sequenceid=40, filesize=16.3 K 2024-11-10T17:22:13,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7047d3abf1e946bfbb6a68770d1e7b07 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7047d3abf1e946bfbb6a68770d1e7b07 2024-11-10T17:22:13,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7047d3abf1e946bfbb6a68770d1e7b07, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T17:22:13,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/48f876ea596144f09741acc90b907c87 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48f876ea596144f09741acc90b907c87 2024-11-10T17:22:13,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48f876ea596144f09741acc90b907c87, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T17:22:13,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 86036b2e5b1a6747cefd43fa6b653d49 in 381ms, sequenceid=40, compaction requested=false 2024-11-10T17:22:13,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:13,314 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:13,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:13,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/4ef6eee48ef3422688760e65537e35d3 is 50, key is test_row_0/A:col10/1731259333301/Put/seqid=0 2024-11-10T17:22:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T17:22:13,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741843_1019 (size=19021) 2024-11-10T17:22:13,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:13,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259393548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259393548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259393568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259393555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259393575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259393702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259393703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259393704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259393705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259393705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/4ef6eee48ef3422688760e65537e35d3 2024-11-10T17:22:13,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/8d6f88f828e942a8928f673fec51869f is 50, key is test_row_0/B:col10/1731259333301/Put/seqid=0 2024-11-10T17:22:13,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741844_1020 (size=12001) 2024-11-10T17:22:13,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/8d6f88f828e942a8928f673fec51869f 2024-11-10T17:22:13,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/48edaf96184e467aaabc5a53bfbf6c7b is 50, key is test_row_0/C:col10/1731259333301/Put/seqid=0 2024-11-10T17:22:13,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:13,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259393923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259393927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259393927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259393932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:13,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741845_1021 (size=12001) 2024-11-10T17:22:13,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259393932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:14,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,240 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T17:22:14,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259394243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259394244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259394244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259394244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259394252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/48edaf96184e467aaabc5a53bfbf6c7b 2024-11-10T17:22:14,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/4ef6eee48ef3422688760e65537e35d3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4ef6eee48ef3422688760e65537e35d3 2024-11-10T17:22:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T17:22:14,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4ef6eee48ef3422688760e65537e35d3, entries=300, sequenceid=51, filesize=18.6 K 2024-11-10T17:22:14,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/8d6f88f828e942a8928f673fec51869f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8d6f88f828e942a8928f673fec51869f 2024-11-10T17:22:14,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8d6f88f828e942a8928f673fec51869f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T17:22:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/48edaf96184e467aaabc5a53bfbf6c7b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48edaf96184e467aaabc5a53bfbf6c7b 2024-11-10T17:22:14,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:14,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:14,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:14,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48edaf96184e467aaabc5a53bfbf6c7b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T17:22:14,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 86036b2e5b1a6747cefd43fa6b653d49 in 1103ms, sequenceid=51, compaction requested=true 2024-11-10T17:22:14,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:14,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:14,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:14,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:14,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:14,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:14,424 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:14,424 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:14,429 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:14,431 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:14,431 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,432 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/2b7008fb89674c3dab5e982a7dcde82e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7047d3abf1e946bfbb6a68770d1e7b07, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8d6f88f828e942a8928f673fec51869f] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.2 K 2024-11-10T17:22:14,434 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:14,434 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:14,434 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b7008fb89674c3dab5e982a7dcde82e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731259332170 2024-11-10T17:22:14,434 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,435 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/bc18cf810cf54772a7193496816d258c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/63ab4a29ea3945c59552012c1e332568, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4ef6eee48ef3422688760e65537e35d3] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=46.6 K 2024-11-10T17:22:14,435 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7047d3abf1e946bfbb6a68770d1e7b07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259332485 2024-11-10T17:22:14,439 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc18cf810cf54772a7193496816d258c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731259332170 2024-11-10T17:22:14,439 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6f88f828e942a8928f673fec51869f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259332950 2024-11-10T17:22:14,444 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63ab4a29ea3945c59552012c1e332568, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259332481 2024-11-10T17:22:14,445 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef6eee48ef3422688760e65537e35d3, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259332916 2024-11-10T17:22:14,496 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#9 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:14,497 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/1b8a0f1e9d0244c2a032e13ba69525e7 is 50, key is test_row_0/B:col10/1731259333301/Put/seqid=0 2024-11-10T17:22:14,499 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#10 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:14,500 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/53f9f1e37d9a4176b75f6666697368ba is 50, key is test_row_0/A:col10/1731259333301/Put/seqid=0 2024-11-10T17:22:14,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741846_1022 (size=12104) 2024-11-10T17:22:14,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741847_1023 (size=12104) 2024-11-10T17:22:14,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T17:22:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,558 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:22:14,563 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/53f9f1e37d9a4176b75f6666697368ba as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/53f9f1e37d9a4176b75f6666697368ba 2024-11-10T17:22:14,565 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/1b8a0f1e9d0244c2a032e13ba69525e7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/1b8a0f1e9d0244c2a032e13ba69525e7 2024-11-10T17:22:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/047f11a9977e44a48cbd5b3af96e8741 is 50, key is test_row_0/A:col10/1731259333563/Put/seqid=0 2024-11-10T17:22:14,596 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 1b8a0f1e9d0244c2a032e13ba69525e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:14,596 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:14,597 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into 53f9f1e37d9a4176b75f6666697368ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:14,597 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:14,600 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259334419; duration=0sec 2024-11-10T17:22:14,600 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:14,601 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:14,601 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:14,601 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259334423; duration=0sec 2024-11-10T17:22:14,602 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:14,602 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:14,603 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:14,604 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:14,604 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,604 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/719fc7ee3ab4450a8d36365539d8fe64, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48f876ea596144f09741acc90b907c87, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48edaf96184e467aaabc5a53bfbf6c7b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.2 K 2024-11-10T17:22:14,605 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719fc7ee3ab4450a8d36365539d8fe64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731259332170 2024-11-10T17:22:14,606 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f876ea596144f09741acc90b907c87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259332485 2024-11-10T17:22:14,612 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48edaf96184e467aaabc5a53bfbf6c7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259332950 2024-11-10T17:22:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741848_1024 (size=12001) 2024-11-10T17:22:14,623 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/047f11a9977e44a48cbd5b3af96e8741 2024-11-10T17:22:14,648 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:14,649 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/53d3bc0b5dab4df5a6dfdb5dd94443f2 is 50, key is test_row_0/C:col10/1731259333301/Put/seqid=0 2024-11-10T17:22:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/8af189e2c4e14a7caecdea02e0773cb9 is 50, key is test_row_0/B:col10/1731259333563/Put/seqid=0 2024-11-10T17:22:14,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741849_1025 (size=12104) 2024-11-10T17:22:14,700 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/53d3bc0b5dab4df5a6dfdb5dd94443f2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/53d3bc0b5dab4df5a6dfdb5dd94443f2 2024-11-10T17:22:14,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741850_1026 (size=12001) 2024-11-10T17:22:14,720 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into 53d3bc0b5dab4df5a6dfdb5dd94443f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:14,720 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:14,720 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259334423; duration=0sec 2024-11-10T17:22:14,721 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:14,721 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:14,722 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/8af189e2c4e14a7caecdea02e0773cb9 2024-11-10T17:22:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/55da5ebb30b04eb486c285673a1cd318 is 50, key is test_row_0/C:col10/1731259333563/Put/seqid=0 2024-11-10T17:22:14,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:14,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741851_1027 (size=12001) 2024-11-10T17:22:14,797 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/55da5ebb30b04eb486c285673a1cd318 2024-11-10T17:22:14,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259394786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259394792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259394797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259394797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/047f11a9977e44a48cbd5b3af96e8741 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/047f11a9977e44a48cbd5b3af96e8741 2024-11-10T17:22:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259394803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:14,823 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/047f11a9977e44a48cbd5b3af96e8741, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T17:22:14,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/8af189e2c4e14a7caecdea02e0773cb9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8af189e2c4e14a7caecdea02e0773cb9 2024-11-10T17:22:14,840 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8af189e2c4e14a7caecdea02e0773cb9, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T17:22:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/55da5ebb30b04eb486c285673a1cd318 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/55da5ebb30b04eb486c285673a1cd318 2024-11-10T17:22:14,856 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/55da5ebb30b04eb486c285673a1cd318, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T17:22:14,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 86036b2e5b1a6747cefd43fa6b653d49 in 300ms, sequenceid=76, compaction requested=false 2024-11-10T17:22:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T17:22:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T17:22:14,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T17:22:14,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6430 sec 2024-11-10T17:22:14,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.6700 sec 2024-11-10T17:22:14,903 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T17:22:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:14,932 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:14,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T17:22:14,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3321df543c184d5fbb48a01b33ef6d8b is 50, key is test_row_0/A:col10/1731259334923/Put/seqid=0 2024-11-10T17:22:14,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741852_1028 (size=16681) 2024-11-10T17:22:15,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3321df543c184d5fbb48a01b33ef6d8b 2024-11-10T17:22:15,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259394998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259395002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259395016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259395016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259395016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/10ab73e493b54802b5c2d0dfadec4055 is 50, key is test_row_0/B:col10/1731259334923/Put/seqid=0 2024-11-10T17:22:15,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741853_1029 (size=12001) 2024-11-10T17:22:15,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/10ab73e493b54802b5c2d0dfadec4055 2024-11-10T17:22:15,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/7d18906bf04d4f65bf3c847826beddba is 50, key is test_row_0/C:col10/1731259334923/Put/seqid=0 2024-11-10T17:22:15,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259395122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259395132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259395133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259395133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259395133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741854_1030 (size=12001) 2024-11-10T17:22:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259395332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259395345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259395353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259395354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259395354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/7d18906bf04d4f65bf3c847826beddba 2024-11-10T17:22:15,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3321df543c184d5fbb48a01b33ef6d8b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3321df543c184d5fbb48a01b33ef6d8b 2024-11-10T17:22:15,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3321df543c184d5fbb48a01b33ef6d8b, entries=250, sequenceid=94, filesize=16.3 K 2024-11-10T17:22:15,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/10ab73e493b54802b5c2d0dfadec4055 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/10ab73e493b54802b5c2d0dfadec4055 2024-11-10T17:22:15,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/10ab73e493b54802b5c2d0dfadec4055, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T17:22:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/7d18906bf04d4f65bf3c847826beddba as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7d18906bf04d4f65bf3c847826beddba 2024-11-10T17:22:15,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259395641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7d18906bf04d4f65bf3c847826beddba, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T17:22:15,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 86036b2e5b1a6747cefd43fa6b653d49 in 726ms, sequenceid=94, compaction requested=true 2024-11-10T17:22:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:15,659 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:15,660 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:15,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:15,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:15,662 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:15,662 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:15,662 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:15,663 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/53f9f1e37d9a4176b75f6666697368ba, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/047f11a9977e44a48cbd5b3af96e8741, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3321df543c184d5fbb48a01b33ef6d8b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=39.8 K 2024-11-10T17:22:15,663 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:15,663 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:15,664 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:15,664 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/1b8a0f1e9d0244c2a032e13ba69525e7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8af189e2c4e14a7caecdea02e0773cb9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/10ab73e493b54802b5c2d0dfadec4055] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.3 K 2024-11-10T17:22:15,664 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53f9f1e37d9a4176b75f6666697368ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259332950 2024-11-10T17:22:15,665 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8a0f1e9d0244c2a032e13ba69525e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259332950 2024-11-10T17:22:15,666 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047f11a9977e44a48cbd5b3af96e8741, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259333536 2024-11-10T17:22:15,666 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8af189e2c4e14a7caecdea02e0773cb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259333536 2024-11-10T17:22:15,666 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3321df543c184d5fbb48a01b33ef6d8b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259334775 2024-11-10T17:22:15,667 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 10ab73e493b54802b5c2d0dfadec4055, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259334775 2024-11-10T17:22:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:15,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:22:15,700 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:15,701 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/d557f75b037f42a6a5f20b0da6795392 is 50, key is test_row_0/B:col10/1731259334923/Put/seqid=0 2024-11-10T17:22:15,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/861ec654e5804f5db3120180ff2099cd is 50, key is test_row_0/A:col10/1731259335687/Put/seqid=0 2024-11-10T17:22:15,706 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:15,707 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/6a26b13261e74a98b156b1cdfa1f2c83 is 50, key is test_row_0/A:col10/1731259334923/Put/seqid=0 2024-11-10T17:22:15,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259395712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259395717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259395725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259395728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741855_1031 (size=12207) 2024-11-10T17:22:15,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741856_1032 (size=12001) 2024-11-10T17:22:15,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/861ec654e5804f5db3120180ff2099cd 2024-11-10T17:22:15,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741857_1033 (size=12207) 2024-11-10T17:22:15,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/06852bc4c0a449659e54dfefa618f4a7 is 50, key is test_row_0/B:col10/1731259335687/Put/seqid=0 2024-11-10T17:22:15,834 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/6a26b13261e74a98b156b1cdfa1f2c83 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/6a26b13261e74a98b156b1cdfa1f2c83 2024-11-10T17:22:15,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259395832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259395842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259395843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:15,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259395843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:15,854 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into 6a26b13261e74a98b156b1cdfa1f2c83(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:15,854 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:15,854 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259335659; duration=0sec 2024-11-10T17:22:15,854 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:15,855 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:15,855 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:15,857 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:15,857 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:15,857 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:15,858 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/53d3bc0b5dab4df5a6dfdb5dd94443f2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/55da5ebb30b04eb486c285673a1cd318, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7d18906bf04d4f65bf3c847826beddba] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.3 K 2024-11-10T17:22:15,859 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53d3bc0b5dab4df5a6dfdb5dd94443f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259332950 2024-11-10T17:22:15,859 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55da5ebb30b04eb486c285673a1cd318, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259333536 2024-11-10T17:22:15,860 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d18906bf04d4f65bf3c847826beddba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259334775 2024-11-10T17:22:15,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741858_1034 (size=12001) 2024-11-10T17:22:15,901 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:15,902 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/027fa7cdb7814b68aab4da453546ab2e is 50, key is test_row_0/C:col10/1731259334923/Put/seqid=0 2024-11-10T17:22:15,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/06852bc4c0a449659e54dfefa618f4a7 2024-11-10T17:22:15,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741859_1035 (size=12207) 2024-11-10T17:22:15,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/d5b2ba02e4fa4795845072f958cbd830 is 50, key is test_row_0/C:col10/1731259335687/Put/seqid=0 2024-11-10T17:22:15,960 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/027fa7cdb7814b68aab4da453546ab2e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/027fa7cdb7814b68aab4da453546ab2e 2024-11-10T17:22:15,976 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into 027fa7cdb7814b68aab4da453546ab2e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:15,976 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:15,976 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259335659; duration=0sec 2024-11-10T17:22:15,976 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:15,976 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:15,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741860_1036 (size=12001) 2024-11-10T17:22:15,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/d5b2ba02e4fa4795845072f958cbd830 2024-11-10T17:22:16,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/861ec654e5804f5db3120180ff2099cd as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/861ec654e5804f5db3120180ff2099cd 2024-11-10T17:22:16,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/861ec654e5804f5db3120180ff2099cd, entries=150, sequenceid=120, filesize=11.7 K 2024-11-10T17:22:16,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/06852bc4c0a449659e54dfefa618f4a7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/06852bc4c0a449659e54dfefa618f4a7 2024-11-10T17:22:16,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/06852bc4c0a449659e54dfefa618f4a7, entries=150, sequenceid=120, filesize=11.7 K 2024-11-10T17:22:16,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/d5b2ba02e4fa4795845072f958cbd830 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/d5b2ba02e4fa4795845072f958cbd830 2024-11-10T17:22:16,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/d5b2ba02e4fa4795845072f958cbd830, entries=150, sequenceid=120, filesize=11.7 K 2024-11-10T17:22:16,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 86036b2e5b1a6747cefd43fa6b653d49 in 358ms, sequenceid=120, compaction requested=false 2024-11-10T17:22:16,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:16,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:16,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:22:16,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/69f55c33687843b4ad9e48480b733218 is 50, key is test_row_0/A:col10/1731259336055/Put/seqid=0 2024-11-10T17:22:16,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741861_1037 (size=12151) 2024-11-10T17:22:16,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/69f55c33687843b4ad9e48480b733218 2024-11-10T17:22:16,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/439b26c6cef14a5bb5900739309f6bfa is 50, key is test_row_0/B:col10/1731259336055/Put/seqid=0 2024-11-10T17:22:16,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259396136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259396151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259396151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259396155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259396162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,176 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/d557f75b037f42a6a5f20b0da6795392 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d557f75b037f42a6a5f20b0da6795392 2024-11-10T17:22:16,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741862_1038 (size=12151) 2024-11-10T17:22:16,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/439b26c6cef14a5bb5900739309f6bfa 2024-11-10T17:22:16,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/dfd61b95230748f59a40f0889603084f is 50, key is test_row_0/C:col10/1731259336055/Put/seqid=0 2024-11-10T17:22:16,202 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into d557f75b037f42a6a5f20b0da6795392(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:16,202 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:16,202 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259335659; duration=0sec 2024-11-10T17:22:16,202 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:16,202 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:16,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741863_1039 (size=12151) 2024-11-10T17:22:16,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/dfd61b95230748f59a40f0889603084f 2024-11-10T17:22:16,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/69f55c33687843b4ad9e48480b733218 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/69f55c33687843b4ad9e48480b733218 2024-11-10T17:22:16,249 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T17:22:16,250 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T17:22:16,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/69f55c33687843b4ad9e48480b733218, entries=150, sequenceid=136, filesize=11.9 K 2024-11-10T17:22:16,256 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T17:22:16,257 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T17:22:16,257 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T17:22:16,257 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T17:22:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/439b26c6cef14a5bb5900739309f6bfa as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/439b26c6cef14a5bb5900739309f6bfa 2024-11-10T17:22:16,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T17:22:16,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T17:22:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259396262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/439b26c6cef14a5bb5900739309f6bfa, entries=150, sequenceid=136, filesize=11.9 K 2024-11-10T17:22:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259396262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259396262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/dfd61b95230748f59a40f0889603084f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dfd61b95230748f59a40f0889603084f 2024-11-10T17:22:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259396263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dfd61b95230748f59a40f0889603084f, entries=150, sequenceid=136, filesize=11.9 K 2024-11-10T17:22:16,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 86036b2e5b1a6747cefd43fa6b653d49 in 219ms, sequenceid=136, compaction requested=true 2024-11-10T17:22:16,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:16,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:16,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:16,286 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:16,286 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:16,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:16,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:16,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:16,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:16,288 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:16,288 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:16,288 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,288 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d557f75b037f42a6a5f20b0da6795392, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/06852bc4c0a449659e54dfefa618f4a7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/439b26c6cef14a5bb5900739309f6bfa] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.5 K 2024-11-10T17:22:16,289 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:16,289 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:16,289 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,290 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/6a26b13261e74a98b156b1cdfa1f2c83, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/861ec654e5804f5db3120180ff2099cd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/69f55c33687843b4ad9e48480b733218] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.5 K 2024-11-10T17:22:16,290 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d557f75b037f42a6a5f20b0da6795392, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259334775 2024-11-10T17:22:16,290 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a26b13261e74a98b156b1cdfa1f2c83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259334775 2024-11-10T17:22:16,291 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 06852bc4c0a449659e54dfefa618f4a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731259335667 2024-11-10T17:22:16,293 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 439b26c6cef14a5bb5900739309f6bfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259336055 2024-11-10T17:22:16,293 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 861ec654e5804f5db3120180ff2099cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731259335667 2024-11-10T17:22:16,294 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69f55c33687843b4ad9e48480b733218, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259336055 2024-11-10T17:22:16,330 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:16,331 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/fc9981e1382d4304b3821e1760df75a3 is 50, key is test_row_0/B:col10/1731259336055/Put/seqid=0 2024-11-10T17:22:16,344 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:16,346 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/f105ce5e89194fada59333825b396c5d is 50, key is test_row_0/A:col10/1731259336055/Put/seqid=0 2024-11-10T17:22:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T17:22:16,386 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:16,394 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T17:22:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T17:22:16,398 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:16,399 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:16,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:16,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741864_1040 (size=12459) 2024-11-10T17:22:16,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741865_1041 (size=12459) 2024-11-10T17:22:16,435 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/f105ce5e89194fada59333825b396c5d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/f105ce5e89194fada59333825b396c5d 2024-11-10T17:22:16,436 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/fc9981e1382d4304b3821e1760df75a3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fc9981e1382d4304b3821e1760df75a3 2024-11-10T17:22:16,453 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into f105ce5e89194fada59333825b396c5d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:16,454 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:16,454 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259336286; duration=0sec 2024-11-10T17:22:16,454 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:16,454 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:16,454 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:16,455 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into fc9981e1382d4304b3821e1760df75a3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:16,455 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:16,455 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259336286; duration=0sec 2024-11-10T17:22:16,456 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:16,456 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:16,459 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:16,459 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:16,459 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,459 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/027fa7cdb7814b68aab4da453546ab2e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/d5b2ba02e4fa4795845072f958cbd830, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dfd61b95230748f59a40f0889603084f] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.5 K 2024-11-10T17:22:16,460 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027fa7cdb7814b68aab4da453546ab2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259334775 2024-11-10T17:22:16,462 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b2ba02e4fa4795845072f958cbd830, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731259335667 2024-11-10T17:22:16,462 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfd61b95230748f59a40f0889603084f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259336055 2024-11-10T17:22:16,481 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:16,482 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/bd3b2a26c0604531bf2627f5b5a31b3f is 50, key is test_row_0/C:col10/1731259336055/Put/seqid=0 2024-11-10T17:22:16,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:16,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:22:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T17:22:16,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/89d2670e3b14482ebde63250230851d9 is 50, key is test_row_0/A:col10/1731259336483/Put/seqid=0 2024-11-10T17:22:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259396512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259396514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741866_1042 (size=12459) 2024-11-10T17:22:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259396516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259396525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741867_1043 (size=12151) 2024-11-10T17:22:16,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/89d2670e3b14482ebde63250230851d9 2024-11-10T17:22:16,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/97477dd21fd94f5aadfa0e914a6780af is 50, key is test_row_0/B:col10/1731259336483/Put/seqid=0 2024-11-10T17:22:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741868_1044 (size=12151) 2024-11-10T17:22:16,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/97477dd21fd94f5aadfa0e914a6780af 2024-11-10T17:22:16,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259396633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259396633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259396642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259396642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:16,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/072f32936e6e4fb4831f39ea7645fef3 is 50, key is test_row_0/C:col10/1731259336483/Put/seqid=0 2024-11-10T17:22:16,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741869_1045 (size=12151) 2024-11-10T17:22:16,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/072f32936e6e4fb4831f39ea7645fef3 2024-11-10T17:22:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/89d2670e3b14482ebde63250230851d9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/89d2670e3b14482ebde63250230851d9 2024-11-10T17:22:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T17:22:16,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/89d2670e3b14482ebde63250230851d9, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T17:22:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/97477dd21fd94f5aadfa0e914a6780af as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/97477dd21fd94f5aadfa0e914a6780af 2024-11-10T17:22:16,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/97477dd21fd94f5aadfa0e914a6780af, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T17:22:16,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/072f32936e6e4fb4831f39ea7645fef3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/072f32936e6e4fb4831f39ea7645fef3 2024-11-10T17:22:16,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/072f32936e6e4fb4831f39ea7645fef3, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T17:22:16,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 86036b2e5b1a6747cefd43fa6b653d49 in 265ms, sequenceid=164, compaction requested=false 2024-11-10T17:22:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:16,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:16,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:22:16,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/ebb180bf761c4e3faae2b310d212c91f is 50, key is test_row_0/A:col10/1731259336490/Put/seqid=0 2024-11-10T17:22:16,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:16,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:16,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:16,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:16,939 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/bd3b2a26c0604531bf2627f5b5a31b3f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/bd3b2a26c0604531bf2627f5b5a31b3f 2024-11-10T17:22:16,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741870_1046 (size=14541) 2024-11-10T17:22:17,005 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into bd3b2a26c0604531bf2627f5b5a31b3f(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:17,005 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:17,005 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259336287; duration=0sec 2024-11-10T17:22:17,005 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:17,005 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T17:22:17,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:17,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259397059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259397077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259397082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259397082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259397182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259397182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,197 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 86036b2e5b1a6747cefd43fa6b653d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259397196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259397202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259397202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:17,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:17,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/ebb180bf761c4e3faae2b310d212c91f 2024-11-10T17:22:17,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/3218a4e0b82241edae4e1955dd9ea81b is 50, key is test_row_0/B:col10/1731259336490/Put/seqid=0 2024-11-10T17:22:17,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259397391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259397412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259397412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259397413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741871_1047 (size=12151) 2024-11-10T17:22:17,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/3218a4e0b82241edae4e1955dd9ea81b 2024-11-10T17:22:17,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/403a3397bce14f19a40043ccd8933064 is 50, key is test_row_0/C:col10/1731259336490/Put/seqid=0 2024-11-10T17:22:17,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741872_1048 (size=12151) 2024-11-10T17:22:17,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/403a3397bce14f19a40043ccd8933064 2024-11-10T17:22:17,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/ebb180bf761c4e3faae2b310d212c91f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ebb180bf761c4e3faae2b310d212c91f 2024-11-10T17:22:17,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ebb180bf761c4e3faae2b310d212c91f, entries=200, sequenceid=176, filesize=14.2 K 2024-11-10T17:22:17,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/3218a4e0b82241edae4e1955dd9ea81b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3218a4e0b82241edae4e1955dd9ea81b 2024-11-10T17:22:17,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3218a4e0b82241edae4e1955dd9ea81b, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T17:22:17,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/403a3397bce14f19a40043ccd8933064 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/403a3397bce14f19a40043ccd8933064 2024-11-10T17:22:17,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/403a3397bce14f19a40043ccd8933064, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T17:22:17,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 86036b2e5b1a6747cefd43fa6b653d49 in 670ms, sequenceid=176, compaction requested=true 2024-11-10T17:22:17,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:17,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:17,529 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:17,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:17,530 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T17:22:17,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:17,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:17,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:17,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:17,533 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:17,533 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:17,534 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:17,534 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,534 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:17,535 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,535 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/f105ce5e89194fada59333825b396c5d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/89d2670e3b14482ebde63250230851d9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ebb180bf761c4e3faae2b310d212c91f] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=38.2 K 2024-11-10T17:22:17,534 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fc9981e1382d4304b3821e1760df75a3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/97477dd21fd94f5aadfa0e914a6780af, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3218a4e0b82241edae4e1955dd9ea81b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.9 K 2024-11-10T17:22:17,536 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting fc9981e1382d4304b3821e1760df75a3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259336055 2024-11-10T17:22:17,536 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting f105ce5e89194fada59333825b396c5d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259336055 2024-11-10T17:22:17,537 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 97477dd21fd94f5aadfa0e914a6780af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731259336146 2024-11-10T17:22:17,537 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d2670e3b14482ebde63250230851d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731259336146 2024-11-10T17:22:17,538 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3218a4e0b82241edae4e1955dd9ea81b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731259336490 2024-11-10T17:22:17,538 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb180bf761c4e3faae2b310d212c91f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731259336490 2024-11-10T17:22:17,566 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:17,566 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:17,577 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/9c601736d44e45168d992390e9c78d60 is 50, key is test_row_0/B:col10/1731259336490/Put/seqid=0 2024-11-10T17:22:17,577 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/a03dee0eae11411b97292bd118439ef5 is 50, key is test_row_0/A:col10/1731259336490/Put/seqid=0 2024-11-10T17:22:17,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741873_1049 (size=12561) 2024-11-10T17:22:17,628 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/9c601736d44e45168d992390e9c78d60 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9c601736d44e45168d992390e9c78d60 2024-11-10T17:22:17,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741874_1050 (size=12561) 2024-11-10T17:22:17,644 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 9c601736d44e45168d992390e9c78d60(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:17,644 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:17,645 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259337529; duration=0sec 2024-11-10T17:22:17,645 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:17,645 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:17,645 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:17,654 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:17,654 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:17,654 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,655 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/bd3b2a26c0604531bf2627f5b5a31b3f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/072f32936e6e4fb4831f39ea7645fef3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/403a3397bce14f19a40043ccd8933064] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=35.9 K 2024-11-10T17:22:17,656 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3b2a26c0604531bf2627f5b5a31b3f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259336055 2024-11-10T17:22:17,660 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 072f32936e6e4fb4831f39ea7645fef3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731259336146 2024-11-10T17:22:17,662 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 403a3397bce14f19a40043ccd8933064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731259336490 2024-11-10T17:22:17,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T17:22:17,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:17,687 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:17,687 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:22:17,689 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/a8abf8259d874930b405db23aa0f92db is 50, key is test_row_0/C:col10/1731259336490/Put/seqid=0 2024-11-10T17:22:17,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:17,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3bb27e5037464fad9f97a1dc0e44f000 is 50, key is test_row_0/A:col10/1731259337074/Put/seqid=0 2024-11-10T17:22:17,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259397732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259397744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259397745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259397747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741875_1051 (size=12561) 2024-11-10T17:22:17,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741876_1052 (size=12151) 2024-11-10T17:22:17,827 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3bb27e5037464fad9f97a1dc0e44f000 2024-11-10T17:22:17,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259397852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259397862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259397863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259397862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:17,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7b6311a3285c40a383b54e5bfa9f57b3 is 50, key is test_row_0/B:col10/1731259337074/Put/seqid=0 2024-11-10T17:22:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:17,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741877_1053 (size=12151) 2024-11-10T17:22:17,921 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7b6311a3285c40a383b54e5bfa9f57b3 2024-11-10T17:22:17,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/ffa2b105f0cb46abbfa31df2113e884e is 50, key is test_row_0/C:col10/1731259337074/Put/seqid=0 2024-11-10T17:22:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741878_1054 (size=12151) 2024-11-10T17:22:17,981 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/ffa2b105f0cb46abbfa31df2113e884e 2024-11-10T17:22:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3bb27e5037464fad9f97a1dc0e44f000 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3bb27e5037464fad9f97a1dc0e44f000 2024-11-10T17:22:18,004 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3bb27e5037464fad9f97a1dc0e44f000, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T17:22:18,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7b6311a3285c40a383b54e5bfa9f57b3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7b6311a3285c40a383b54e5bfa9f57b3 2024-11-10T17:22:18,023 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7b6311a3285c40a383b54e5bfa9f57b3, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T17:22:18,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/ffa2b105f0cb46abbfa31df2113e884e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/ffa2b105f0cb46abbfa31df2113e884e 2024-11-10T17:22:18,033 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/ffa2b105f0cb46abbfa31df2113e884e, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T17:22:18,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 86036b2e5b1a6747cefd43fa6b653d49 in 354ms, sequenceid=204, compaction requested=false 2024-11-10T17:22:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T17:22:18,043 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/a03dee0eae11411b97292bd118439ef5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/a03dee0eae11411b97292bd118439ef5 2024-11-10T17:22:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T17:22:18,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T17:22:18,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6400 sec 2024-11-10T17:22:18,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6560 sec 2024-11-10T17:22:18,058 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into a03dee0eae11411b97292bd118439ef5(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:18,058 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:18,058 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259337529; duration=0sec 2024-11-10T17:22:18,058 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:18,058 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:18,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:18,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:22:18,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/82aa14a628474b7f8eab80bdb32c319d is 50, key is test_row_0/A:col10/1731259337742/Put/seqid=0 2024-11-10T17:22:18,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741879_1055 (size=14541) 2024-11-10T17:22:18,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/82aa14a628474b7f8eab80bdb32c319d 2024-11-10T17:22:18,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259398149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259398151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/ba46d22105ad43f593468f00f3be66dd is 50, key is test_row_0/B:col10/1731259337742/Put/seqid=0 2024-11-10T17:22:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259398154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259398155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,198 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/a8abf8259d874930b405db23aa0f92db as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a8abf8259d874930b405db23aa0f92db 2024-11-10T17:22:18,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741880_1056 (size=12151) 2024-11-10T17:22:18,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/ba46d22105ad43f593468f00f3be66dd 2024-11-10T17:22:18,213 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into a8abf8259d874930b405db23aa0f92db(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:18,213 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:18,213 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259337532; duration=0sec 2024-11-10T17:22:18,213 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:18,213 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:18,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/18bfb30b66d449b6bc35a34a11ce8874 is 50, key is test_row_0/C:col10/1731259337742/Put/seqid=0 2024-11-10T17:22:18,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741881_1057 (size=12151) 2024-11-10T17:22:18,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/18bfb30b66d449b6bc35a34a11ce8874 2024-11-10T17:22:18,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/82aa14a628474b7f8eab80bdb32c319d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/82aa14a628474b7f8eab80bdb32c319d 2024-11-10T17:22:18,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/82aa14a628474b7f8eab80bdb32c319d, entries=200, sequenceid=218, filesize=14.2 K 2024-11-10T17:22:18,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/ba46d22105ad43f593468f00f3be66dd as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/ba46d22105ad43f593468f00f3be66dd 2024-11-10T17:22:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259398262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259398262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/ba46d22105ad43f593468f00f3be66dd, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T17:22:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259398271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/18bfb30b66d449b6bc35a34a11ce8874 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/18bfb30b66d449b6bc35a34a11ce8874 2024-11-10T17:22:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259398271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/18bfb30b66d449b6bc35a34a11ce8874, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T17:22:18,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 86036b2e5b1a6747cefd43fa6b653d49 in 211ms, sequenceid=218, compaction requested=true 2024-11-10T17:22:18,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:18,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:18,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:18,288 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:18,288 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:18,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:18,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:18,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:18,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:18,290 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:18,290 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:18,290 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,290 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/a03dee0eae11411b97292bd118439ef5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3bb27e5037464fad9f97a1dc0e44f000, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/82aa14a628474b7f8eab80bdb32c319d] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=38.3 K 2024-11-10T17:22:18,290 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:18,290 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:18,290 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,291 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9c601736d44e45168d992390e9c78d60, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7b6311a3285c40a383b54e5bfa9f57b3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/ba46d22105ad43f593468f00f3be66dd] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=36.0 K 2024-11-10T17:22:18,291 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting a03dee0eae11411b97292bd118439ef5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731259336490 2024-11-10T17:22:18,291 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c601736d44e45168d992390e9c78d60, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731259336490 2024-11-10T17:22:18,292 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bb27e5037464fad9f97a1dc0e44f000, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731259337055 2024-11-10T17:22:18,292 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b6311a3285c40a383b54e5bfa9f57b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731259337055 2024-11-10T17:22:18,293 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82aa14a628474b7f8eab80bdb32c319d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731259337740 2024-11-10T17:22:18,293 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting ba46d22105ad43f593468f00f3be66dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731259337742 2024-11-10T17:22:18,311 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:18,312 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/4515512378b4415bb374b14515fa0240 is 50, key is test_row_0/A:col10/1731259337742/Put/seqid=0 2024-11-10T17:22:18,323 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:18,324 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/03eb247987f747d0995358ba9ea1a5db is 50, key is test_row_0/B:col10/1731259337742/Put/seqid=0 2024-11-10T17:22:18,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741882_1058 (size=12663) 2024-11-10T17:22:18,368 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/4515512378b4415bb374b14515fa0240 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4515512378b4415bb374b14515fa0240 2024-11-10T17:22:18,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741883_1059 (size=12663) 2024-11-10T17:22:18,378 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into 4515512378b4415bb374b14515fa0240(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:18,378 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:18,379 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259338288; duration=0sec 2024-11-10T17:22:18,379 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:18,379 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:18,379 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:18,380 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/03eb247987f747d0995358ba9ea1a5db as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/03eb247987f747d0995358ba9ea1a5db 2024-11-10T17:22:18,381 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:18,381 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:18,381 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,381 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a8abf8259d874930b405db23aa0f92db, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/ffa2b105f0cb46abbfa31df2113e884e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/18bfb30b66d449b6bc35a34a11ce8874] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=36.0 K 2024-11-10T17:22:18,384 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8abf8259d874930b405db23aa0f92db, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731259336490 2024-11-10T17:22:18,385 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffa2b105f0cb46abbfa31df2113e884e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731259337055 2024-11-10T17:22:18,385 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18bfb30b66d449b6bc35a34a11ce8874, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731259337742 2024-11-10T17:22:18,393 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 03eb247987f747d0995358ba9ea1a5db(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:18,393 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:18,394 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259338288; duration=0sec 2024-11-10T17:22:18,395 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:18,395 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:18,400 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:18,400 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/07f9d996a06f4dcfba470389d88289ff is 50, key is test_row_0/C:col10/1731259337742/Put/seqid=0 2024-11-10T17:22:18,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741884_1060 (size=12663) 2024-11-10T17:22:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:18,478 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:18,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:22:18,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259398499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259398499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/734ed7485a754ae091f53772872209fb is 50, key is test_row_0/A:col10/1731259338146/Put/seqid=0 2024-11-10T17:22:18,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259398502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259398502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741885_1061 (size=16931) 2024-11-10T17:22:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T17:22:18,542 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:18,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T17:22:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T17:22:18,549 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:18,550 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:18,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:18,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259398614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259398614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259398614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259398614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T17:22:18,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T17:22:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:18,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259398822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259398832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,835 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/07f9d996a06f4dcfba470389d88289ff as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/07f9d996a06f4dcfba470389d88289ff 2024-11-10T17:22:18,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259398835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259398835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:18,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T17:22:18,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:18,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:18,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:18,862 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into 07f9d996a06f4dcfba470389d88289ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:18,862 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:18,862 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259338289; duration=0sec 2024-11-10T17:22:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T17:22:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:18,864 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:18,864 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:18,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/734ed7485a754ae091f53772872209fb 2024-11-10T17:22:18,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/917de6c43da747078bad395f2b54c9c2 is 50, key is test_row_0/B:col10/1731259338146/Put/seqid=0 2024-11-10T17:22:18,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741886_1062 (size=12151) 2024-11-10T17:22:18,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/917de6c43da747078bad395f2b54c9c2 2024-11-10T17:22:19,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T17:22:19,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:19,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:19,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:19,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4de913fe7f1e424bb999af23c9fa24ff is 50, key is test_row_0/C:col10/1731259338146/Put/seqid=0 2024-11-10T17:22:19,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741887_1063 (size=12151) 2024-11-10T17:22:19,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4de913fe7f1e424bb999af23c9fa24ff 2024-11-10T17:22:19,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/734ed7485a754ae091f53772872209fb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/734ed7485a754ae091f53772872209fb 2024-11-10T17:22:19,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/734ed7485a754ae091f53772872209fb, entries=250, sequenceid=248, filesize=16.5 K 2024-11-10T17:22:19,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/917de6c43da747078bad395f2b54c9c2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/917de6c43da747078bad395f2b54c9c2 2024-11-10T17:22:19,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/917de6c43da747078bad395f2b54c9c2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-10T17:22:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4de913fe7f1e424bb999af23c9fa24ff as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4de913fe7f1e424bb999af23c9fa24ff 2024-11-10T17:22:19,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4de913fe7f1e424bb999af23c9fa24ff, entries=150, sequenceid=248, filesize=11.9 K 2024-11-10T17:22:19,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 86036b2e5b1a6747cefd43fa6b653d49 in 622ms, sequenceid=248, compaction requested=false 2024-11-10T17:22:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:19,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:19,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:19,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/dd5440b93f3c450da5f1684e3a7d9bd8 is 50, key is test_row_0/A:col10/1731259338497/Put/seqid=0 2024-11-10T17:22:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T17:22:19,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T17:22:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:19,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741888_1064 (size=12201) 2024-11-10T17:22:19,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/dd5440b93f3c450da5f1684e3a7d9bd8 2024-11-10T17:22:19,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5feb181a49f04f93bf3a046cb8d47446 is 50, key is test_row_0/B:col10/1731259338497/Put/seqid=0 2024-11-10T17:22:19,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741889_1065 (size=12201) 2024-11-10T17:22:19,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5feb181a49f04f93bf3a046cb8d47446 2024-11-10T17:22:19,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/7a839557aaed472cbdd727a40fda893c is 50, key is test_row_0/C:col10/1731259338497/Put/seqid=0 2024-11-10T17:22:19,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741890_1066 (size=12201) 2024-11-10T17:22:19,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/7a839557aaed472cbdd727a40fda893c 2024-11-10T17:22:19,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/dd5440b93f3c450da5f1684e3a7d9bd8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/dd5440b93f3c450da5f1684e3a7d9bd8 2024-11-10T17:22:19,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/dd5440b93f3c450da5f1684e3a7d9bd8, entries=150, sequenceid=260, filesize=11.9 K 2024-11-10T17:22:19,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5feb181a49f04f93bf3a046cb8d47446 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5feb181a49f04f93bf3a046cb8d47446 2024-11-10T17:22:19,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259399269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259399284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259399284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259399285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5feb181a49f04f93bf3a046cb8d47446, entries=150, sequenceid=260, filesize=11.9 K 2024-11-10T17:22:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259399291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/7a839557aaed472cbdd727a40fda893c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7a839557aaed472cbdd727a40fda893c 2024-11-10T17:22:19,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7a839557aaed472cbdd727a40fda893c, entries=150, sequenceid=260, filesize=11.9 K 2024-11-10T17:22:19,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 86036b2e5b1a6747cefd43fa6b653d49 in 171ms, sequenceid=260, compaction requested=true 2024-11-10T17:22:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:19,313 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:19,313 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:19,316 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:19,316 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:19,316 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:19,316 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,316 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:19,316 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,316 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4515512378b4415bb374b14515fa0240, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/734ed7485a754ae091f53772872209fb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/dd5440b93f3c450da5f1684e3a7d9bd8] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=40.8 K 2024-11-10T17:22:19,316 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/03eb247987f747d0995358ba9ea1a5db, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/917de6c43da747078bad395f2b54c9c2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5feb181a49f04f93bf3a046cb8d47446] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=36.1 K 2024-11-10T17:22:19,317 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4515512378b4415bb374b14515fa0240, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731259337742 2024-11-10T17:22:19,317 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 03eb247987f747d0995358ba9ea1a5db, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731259337742 2024-11-10T17:22:19,317 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734ed7485a754ae091f53772872209fb, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731259338123 2024-11-10T17:22:19,318 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 917de6c43da747078bad395f2b54c9c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731259338123 2024-11-10T17:22:19,318 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5440b93f3c450da5f1684e3a7d9bd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731259338497 2024-11-10T17:22:19,318 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 5feb181a49f04f93bf3a046cb8d47446, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731259338497 2024-11-10T17:22:19,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T17:22:19,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,353 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:22:19,356 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:19,356 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/044d10da5ee1484ba4f3105b2fac77c6 is 50, key is test_row_0/B:col10/1731259338497/Put/seqid=0 2024-11-10T17:22:19,357 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:19,357 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/e44ace6dd4004cb5abfd79b2eb175ca2 is 50, key is test_row_0/A:col10/1731259338497/Put/seqid=0 2024-11-10T17:22:19,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/47370ae15de744429991368cf1fe1451 is 50, key is test_row_0/A:col10/1731259339288/Put/seqid=0 2024-11-10T17:22:19,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:19,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741892_1068 (size=12301) 2024-11-10T17:22:19,419 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/47370ae15de744429991368cf1fe1451 2024-11-10T17:22:19,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259399411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259399413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259399414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259399413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259399415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741891_1067 (size=12815) 2024-11-10T17:22:19,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/a9447c45399548f58b6161b2228a4150 is 50, key is test_row_0/B:col10/1731259339288/Put/seqid=0 2024-11-10T17:22:19,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741893_1069 (size=12815) 2024-11-10T17:22:19,455 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/e44ace6dd4004cb5abfd79b2eb175ca2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/e44ace6dd4004cb5abfd79b2eb175ca2 2024-11-10T17:22:19,464 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/044d10da5ee1484ba4f3105b2fac77c6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/044d10da5ee1484ba4f3105b2fac77c6 2024-11-10T17:22:19,471 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into e44ace6dd4004cb5abfd79b2eb175ca2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:19,471 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:19,472 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259339313; duration=0sec 2024-11-10T17:22:19,473 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:19,473 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:19,473 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:19,477 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:19,477 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:19,478 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:19,478 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/07f9d996a06f4dcfba470389d88289ff, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4de913fe7f1e424bb999af23c9fa24ff, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7a839557aaed472cbdd727a40fda893c] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=36.1 K 2024-11-10T17:22:19,478 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f9d996a06f4dcfba470389d88289ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731259337742 2024-11-10T17:22:19,479 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4de913fe7f1e424bb999af23c9fa24ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731259338123 2024-11-10T17:22:19,479 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a839557aaed472cbdd727a40fda893c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731259338497 2024-11-10T17:22:19,480 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 044d10da5ee1484ba4f3105b2fac77c6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:19,480 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:19,480 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259339313; duration=0sec 2024-11-10T17:22:19,481 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:19,481 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:19,493 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#58 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:19,494 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/dc7b8b2dedb343e3aaa259392b2eac7e is 50, key is test_row_0/C:col10/1731259338497/Put/seqid=0 2024-11-10T17:22:19,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741894_1070 (size=12301) 2024-11-10T17:22:19,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741895_1071 (size=12815) 2024-11-10T17:22:19,529 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/dc7b8b2dedb343e3aaa259392b2eac7e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dc7b8b2dedb343e3aaa259392b2eac7e 2024-11-10T17:22:19,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259399532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259399533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259399533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259399533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,539 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into dc7b8b2dedb343e3aaa259392b2eac7e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:19,539 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:19,539 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259339313; duration=0sec 2024-11-10T17:22:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,539 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,539 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:19,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259399533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T17:22:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259399742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259399743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259399745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259399746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259399752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:19,915 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/a9447c45399548f58b6161b2228a4150 2024-11-10T17:22:19,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/99d48065f9b04b4cb977d60e157c747b is 50, key is test_row_0/C:col10/1731259339288/Put/seqid=0 2024-11-10T17:22:19,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741896_1072 (size=12301) 2024-11-10T17:22:19,941 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/99d48065f9b04b4cb977d60e157c747b 2024-11-10T17:22:19,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/47370ae15de744429991368cf1fe1451 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/47370ae15de744429991368cf1fe1451 2024-11-10T17:22:19,958 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/47370ae15de744429991368cf1fe1451, entries=150, sequenceid=286, filesize=12.0 K 2024-11-10T17:22:19,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/a9447c45399548f58b6161b2228a4150 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/a9447c45399548f58b6161b2228a4150 2024-11-10T17:22:19,976 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/a9447c45399548f58b6161b2228a4150, entries=150, sequenceid=286, filesize=12.0 K 2024-11-10T17:22:19,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/99d48065f9b04b4cb977d60e157c747b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/99d48065f9b04b4cb977d60e157c747b 2024-11-10T17:22:19,999 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/99d48065f9b04b4cb977d60e157c747b, entries=150, sequenceid=286, filesize=12.0 K 2024-11-10T17:22:20,001 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 86036b2e5b1a6747cefd43fa6b653d49 in 648ms, sequenceid=286, compaction requested=false 2024-11-10T17:22:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T17:22:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T17:22:20,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T17:22:20,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4550 sec 2024-11-10T17:22:20,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4660 sec 2024-11-10T17:22:20,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:20,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:20,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/95c9d14b600e4ab1ac4477677d8f83e6 is 50, key is test_row_0/A:col10/1731259339410/Put/seqid=0 2024-11-10T17:22:20,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741897_1073 (size=17181) 2024-11-10T17:22:20,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/95c9d14b600e4ab1ac4477677d8f83e6 2024-11-10T17:22:20,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/b49310a3c4ae4f228fc3636cdf849333 is 50, key is test_row_0/B:col10/1731259339410/Put/seqid=0 2024-11-10T17:22:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259400106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259400110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259400113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259400115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259400117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741898_1074 (size=12301) 2024-11-10T17:22:20,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/b49310a3c4ae4f228fc3636cdf849333 2024-11-10T17:22:20,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/54f579ed6d204440aa31a8e6d019d834 is 50, key is test_row_0/C:col10/1731259339410/Put/seqid=0 2024-11-10T17:22:20,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741899_1075 (size=12301) 2024-11-10T17:22:20,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/54f579ed6d204440aa31a8e6d019d834 2024-11-10T17:22:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/95c9d14b600e4ab1ac4477677d8f83e6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/95c9d14b600e4ab1ac4477677d8f83e6 2024-11-10T17:22:20,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/95c9d14b600e4ab1ac4477677d8f83e6, entries=250, sequenceid=301, filesize=16.8 K 2024-11-10T17:22:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/b49310a3c4ae4f228fc3636cdf849333 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b49310a3c4ae4f228fc3636cdf849333 2024-11-10T17:22:20,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b49310a3c4ae4f228fc3636cdf849333, entries=150, sequenceid=301, filesize=12.0 K 2024-11-10T17:22:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/54f579ed6d204440aa31a8e6d019d834 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/54f579ed6d204440aa31a8e6d019d834 2024-11-10T17:22:20,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/54f579ed6d204440aa31a8e6d019d834, entries=150, sequenceid=301, filesize=12.0 K 2024-11-10T17:22:20,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 86036b2e5b1a6747cefd43fa6b653d49 in 148ms, sequenceid=301, compaction requested=true 2024-11-10T17:22:20,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:20,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:20,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:20,215 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:20,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:20,215 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:20,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:20,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:20,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:20,216 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:20,216 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:20,216 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:20,216 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:20,216 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:20,216 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/e44ace6dd4004cb5abfd79b2eb175ca2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/47370ae15de744429991368cf1fe1451, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/95c9d14b600e4ab1ac4477677d8f83e6] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=41.3 K 2024-11-10T17:22:20,216 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:20,216 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/044d10da5ee1484ba4f3105b2fac77c6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/a9447c45399548f58b6161b2228a4150, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b49310a3c4ae4f228fc3636cdf849333] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=36.5 K 2024-11-10T17:22:20,217 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e44ace6dd4004cb5abfd79b2eb175ca2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731259338497 2024-11-10T17:22:20,217 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 044d10da5ee1484ba4f3105b2fac77c6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731259338497 2024-11-10T17:22:20,217 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47370ae15de744429991368cf1fe1451, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731259339211 2024-11-10T17:22:20,218 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95c9d14b600e4ab1ac4477677d8f83e6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731259339410 2024-11-10T17:22:20,218 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting a9447c45399548f58b6161b2228a4150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731259339211 2024-11-10T17:22:20,219 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b49310a3c4ae4f228fc3636cdf849333, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731259339410 2024-11-10T17:22:20,249 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:20,250 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:20,250 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/cb0e8727784549e7a97a1fec044cbf7a is 50, key is test_row_0/A:col10/1731259339410/Put/seqid=0 2024-11-10T17:22:20,250 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/fb82b70eba1540ff9b59b9df23a9b37b is 50, key is test_row_0/B:col10/1731259339410/Put/seqid=0 2024-11-10T17:22:20,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741900_1076 (size=13017) 2024-11-10T17:22:20,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741901_1077 (size=13017) 2024-11-10T17:22:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:20,266 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:20,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T17:22:20,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/1f578ce8df7e4caa8e0c2b8128036ffe is 50, key is test_row_0/A:col10/1731259340227/Put/seqid=0 2024-11-10T17:22:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259400273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259400280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259400281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259400282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259400283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741902_1078 (size=12301) 2024-11-10T17:22:20,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259400392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259400392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259400392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259400402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259400402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259400605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259400604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259400611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259400612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259400612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,675 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/cb0e8727784549e7a97a1fec044cbf7a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cb0e8727784549e7a97a1fec044cbf7a 2024-11-10T17:22:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T17:22:20,692 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:20,693 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T17:22:20,696 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:20,697 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:20,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:20,698 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/fb82b70eba1540ff9b59b9df23a9b37b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fb82b70eba1540ff9b59b9df23a9b37b 2024-11-10T17:22:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T17:22:20,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/1f578ce8df7e4caa8e0c2b8128036ffe 2024-11-10T17:22:20,705 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into cb0e8727784549e7a97a1fec044cbf7a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:20,705 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:20,705 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259340214; duration=0sec 2024-11-10T17:22:20,705 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:20,705 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:20,705 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:20,708 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:20,708 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:20,708 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:20,709 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dc7b8b2dedb343e3aaa259392b2eac7e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/99d48065f9b04b4cb977d60e157c747b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/54f579ed6d204440aa31a8e6d019d834] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=36.5 K 2024-11-10T17:22:20,710 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc7b8b2dedb343e3aaa259392b2eac7e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731259338497 2024-11-10T17:22:20,710 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into fb82b70eba1540ff9b59b9df23a9b37b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:20,710 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:20,710 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259340215; duration=0sec 2024-11-10T17:22:20,711 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:20,711 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:20,712 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d48065f9b04b4cb977d60e157c747b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731259339211 2024-11-10T17:22:20,712 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f579ed6d204440aa31a8e6d019d834, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731259339410 2024-11-10T17:22:20,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/0036c84230d24fc9a362a60d25c64524 is 50, key is test_row_0/B:col10/1731259340227/Put/seqid=0 2024-11-10T17:22:20,730 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:20,733 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/c9069b7f304e4130a6f87f1ffed6f5dc is 50, key is test_row_0/C:col10/1731259339410/Put/seqid=0 2024-11-10T17:22:20,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741903_1079 (size=12301) 2024-11-10T17:22:20,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741904_1080 (size=13017) 2024-11-10T17:22:20,786 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/c9069b7f304e4130a6f87f1ffed6f5dc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/c9069b7f304e4130a6f87f1ffed6f5dc 2024-11-10T17:22:20,799 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into c9069b7f304e4130a6f87f1ffed6f5dc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:20,799 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:20,799 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259340215; duration=0sec 2024-11-10T17:22:20,800 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:20,800 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T17:22:20,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T17:22:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:20,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:20,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259400912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259400932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259400932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259400932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:20,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259400933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:21,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T17:22:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T17:22:21,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T17:22:21,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:21,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/0036c84230d24fc9a362a60d25c64524 2024-11-10T17:22:21,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/00ca30f77df14fc6b45913f720a5b10b is 50, key is test_row_0/C:col10/1731259340227/Put/seqid=0 2024-11-10T17:22:21,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741905_1081 (size=12301) 2024-11-10T17:22:21,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T17:22:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T17:22:21,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259401422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:21,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259401442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259401442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259401442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259401443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:21,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T17:22:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/00ca30f77df14fc6b45913f720a5b10b 2024-11-10T17:22:21,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/1f578ce8df7e4caa8e0c2b8128036ffe as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/1f578ce8df7e4caa8e0c2b8128036ffe 2024-11-10T17:22:21,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/1f578ce8df7e4caa8e0c2b8128036ffe, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T17:22:21,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/0036c84230d24fc9a362a60d25c64524 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/0036c84230d24fc9a362a60d25c64524 2024-11-10T17:22:21,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T17:22:21,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:21,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:21,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/0036c84230d24fc9a362a60d25c64524, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T17:22:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/00ca30f77df14fc6b45913f720a5b10b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/00ca30f77df14fc6b45913f720a5b10b 2024-11-10T17:22:21,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/00ca30f77df14fc6b45913f720a5b10b, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T17:22:21,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 86036b2e5b1a6747cefd43fa6b653d49 in 1396ms, sequenceid=328, compaction requested=false 2024-11-10T17:22:21,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:21,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T17:22:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:21,789 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T17:22:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/5a46f8764e904297a66ddf155d52d268 is 50, key is test_row_0/A:col10/1731259340279/Put/seqid=0 2024-11-10T17:22:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T17:22:21,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741906_1082 (size=9857) 2024-11-10T17:22:22,272 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/5a46f8764e904297a66ddf155d52d268 2024-11-10T17:22:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/fe133256ba054dfeb56cd9045e4b6a11 is 50, key is test_row_0/B:col10/1731259340279/Put/seqid=0 2024-11-10T17:22:22,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741907_1083 (size=9857) 2024-11-10T17:22:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:22,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259402481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259402483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259402485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259402482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259402488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259402592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259402602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259402603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259402603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259402602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,742 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/fe133256ba054dfeb56cd9045e4b6a11 2024-11-10T17:22:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/e87a4dab370c40f7b15a9ff956216989 is 50, key is test_row_0/C:col10/1731259340279/Put/seqid=0 2024-11-10T17:22:22,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741908_1084 (size=9857) 2024-11-10T17:22:22,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259402802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259402811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259402811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259402823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259402823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T17:22:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259403112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259403121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259403121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259403132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259403132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,187 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/e87a4dab370c40f7b15a9ff956216989 2024-11-10T17:22:23,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/5a46f8764e904297a66ddf155d52d268 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/5a46f8764e904297a66ddf155d52d268 2024-11-10T17:22:23,209 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/5a46f8764e904297a66ddf155d52d268, entries=100, sequenceid=340, filesize=9.6 K 2024-11-10T17:22:23,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/fe133256ba054dfeb56cd9045e4b6a11 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fe133256ba054dfeb56cd9045e4b6a11 2024-11-10T17:22:23,221 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fe133256ba054dfeb56cd9045e4b6a11, entries=100, sequenceid=340, filesize=9.6 K 2024-11-10T17:22:23,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/e87a4dab370c40f7b15a9ff956216989 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/e87a4dab370c40f7b15a9ff956216989 2024-11-10T17:22:23,244 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/e87a4dab370c40f7b15a9ff956216989, entries=100, sequenceid=340, filesize=9.6 K 2024-11-10T17:22:23,250 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 86036b2e5b1a6747cefd43fa6b653d49 in 1460ms, sequenceid=340, compaction requested=true 2024-11-10T17:22:23,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:23,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:23,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T17:22:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T17:22:23,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T17:22:23,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5630 sec 2024-11-10T17:22:23,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.5730 sec 2024-11-10T17:22:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:23,626 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:23,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-10T17:22:23,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/49ea190fdf70415cb47955d710d0ff22 is 50, key is test_row_0/A:col10/1731259342482/Put/seqid=0 2024-11-10T17:22:23,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741909_1085 (size=12301) 2024-11-10T17:22:23,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259403641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259403642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259403642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259403643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259403651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259403751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259403751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259403961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259403961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/49ea190fdf70415cb47955d710d0ff22 2024-11-10T17:22:24,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c10cc757183d412faac5b324977d6714 is 50, key is test_row_0/B:col10/1731259342482/Put/seqid=0 2024-11-10T17:22:24,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741910_1086 (size=12301) 2024-11-10T17:22:24,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c10cc757183d412faac5b324977d6714 2024-11-10T17:22:24,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/6da3bb94089d4c5e983eadf0ef44d498 is 50, key is test_row_0/C:col10/1731259342482/Put/seqid=0 2024-11-10T17:22:24,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741911_1087 (size=12301) 2024-11-10T17:22:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259404274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259404274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/6da3bb94089d4c5e983eadf0ef44d498 2024-11-10T17:22:24,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/49ea190fdf70415cb47955d710d0ff22 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/49ea190fdf70415cb47955d710d0ff22 2024-11-10T17:22:24,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/49ea190fdf70415cb47955d710d0ff22, entries=150, sequenceid=368, filesize=12.0 K 2024-11-10T17:22:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c10cc757183d412faac5b324977d6714 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c10cc757183d412faac5b324977d6714 2024-11-10T17:22:24,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c10cc757183d412faac5b324977d6714, entries=150, sequenceid=368, filesize=12.0 K 2024-11-10T17:22:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/6da3bb94089d4c5e983eadf0ef44d498 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6da3bb94089d4c5e983eadf0ef44d498 2024-11-10T17:22:24,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6da3bb94089d4c5e983eadf0ef44d498, entries=150, sequenceid=368, filesize=12.0 K 2024-11-10T17:22:24,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 86036b2e5b1a6747cefd43fa6b653d49 in 927ms, sequenceid=368, compaction requested=true 2024-11-10T17:22:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:24,554 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T17:22:24,555 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:24,556 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:24,556 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:24,556 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:24,556 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cb0e8727784549e7a97a1fec044cbf7a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/1f578ce8df7e4caa8e0c2b8128036ffe, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/5a46f8764e904297a66ddf155d52d268, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/49ea190fdf70415cb47955d710d0ff22] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=46.4 K 2024-11-10T17:22:24,557 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:24,557 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb0e8727784549e7a97a1fec044cbf7a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731259339410 2024-11-10T17:22:24,557 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:24,558 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:24,558 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fb82b70eba1540ff9b59b9df23a9b37b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/0036c84230d24fc9a362a60d25c64524, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fe133256ba054dfeb56cd9045e4b6a11, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c10cc757183d412faac5b324977d6714] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=46.4 K 2024-11-10T17:22:24,558 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f578ce8df7e4caa8e0c2b8128036ffe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731259340225 2024-11-10T17:22:24,558 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting fb82b70eba1540ff9b59b9df23a9b37b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731259339410 2024-11-10T17:22:24,558 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a46f8764e904297a66ddf155d52d268, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731259340279 2024-11-10T17:22:24,559 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 0036c84230d24fc9a362a60d25c64524, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731259340225 2024-11-10T17:22:24,559 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49ea190fdf70415cb47955d710d0ff22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731259342480 2024-11-10T17:22:24,559 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting fe133256ba054dfeb56cd9045e4b6a11, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731259340279 2024-11-10T17:22:24,560 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c10cc757183d412faac5b324977d6714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731259342480 2024-11-10T17:22:24,585 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:24,586 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/e27aaf631542450aa57fd549824ec220 is 50, key is test_row_0/B:col10/1731259342482/Put/seqid=0 2024-11-10T17:22:24,588 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:24,589 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/23ec77af2e1642268d46ee53c8fa908b is 50, key is test_row_0/A:col10/1731259342482/Put/seqid=0 2024-11-10T17:22:24,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741912_1088 (size=13153) 2024-11-10T17:22:24,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741913_1089 (size=13153) 2024-11-10T17:22:24,642 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/e27aaf631542450aa57fd549824ec220 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/e27aaf631542450aa57fd549824ec220 2024-11-10T17:22:24,645 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/23ec77af2e1642268d46ee53c8fa908b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/23ec77af2e1642268d46ee53c8fa908b 2024-11-10T17:22:24,655 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into e27aaf631542450aa57fd549824ec220(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:24,655 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:24,655 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=12, startTime=1731259344554; duration=0sec 2024-11-10T17:22:24,656 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:24,656 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:24,656 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:24,657 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into 23ec77af2e1642268d46ee53c8fa908b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:24,657 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:24,657 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=12, startTime=1731259344554; duration=0sec 2024-11-10T17:22:24,657 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:24,657 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:24,663 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:24,663 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:24,664 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:24,664 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/c9069b7f304e4130a6f87f1ffed6f5dc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/00ca30f77df14fc6b45913f720a5b10b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/e87a4dab370c40f7b15a9ff956216989, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6da3bb94089d4c5e983eadf0ef44d498] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=46.4 K 2024-11-10T17:22:24,665 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c9069b7f304e4130a6f87f1ffed6f5dc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731259339410 2024-11-10T17:22:24,666 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ca30f77df14fc6b45913f720a5b10b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731259340225 2024-11-10T17:22:24,666 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e87a4dab370c40f7b15a9ff956216989, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731259340279 2024-11-10T17:22:24,667 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da3bb94089d4c5e983eadf0ef44d498, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731259342480 2024-11-10T17:22:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:24,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:24,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:24,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/cfc23839af2741baa5f42c455c8d6006 is 50, key is test_row_0/A:col10/1731259344666/Put/seqid=0 2024-11-10T17:22:24,699 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741914_1090 (size=12301) 2024-11-10T17:22:24,700 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/979e0e6c020542b697eb22736a5f89a2 is 50, key is test_row_0/C:col10/1731259342482/Put/seqid=0 2024-11-10T17:22:24,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/cfc23839af2741baa5f42c455c8d6006 2024-11-10T17:22:24,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741915_1091 (size=13153) 2024-11-10T17:22:24,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/02af42dd510f43ac8ca32a9fef0a62dc is 50, key is test_row_0/B:col10/1731259344666/Put/seqid=0 2024-11-10T17:22:24,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741916_1092 (size=12301) 2024-11-10T17:22:24,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259404759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259404761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259404763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259404781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259404792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T17:22:24,872 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:24,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T17:22:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259404872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259404872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:24,875 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259404872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,877 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:24,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T17:22:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T17:22:25,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T17:22:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:25,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:25,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259405082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259405082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259405082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,119 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/979e0e6c020542b697eb22736a5f89a2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/979e0e6c020542b697eb22736a5f89a2 2024-11-10T17:22:25,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/02af42dd510f43ac8ca32a9fef0a62dc 2024-11-10T17:22:25,133 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into 979e0e6c020542b697eb22736a5f89a2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:25,134 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:25,134 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=12, startTime=1731259344554; duration=0sec 2024-11-10T17:22:25,134 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:25,134 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:25,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/94e2af5f320d4a829cc2e1df6686e580 is 50, key is test_row_0/C:col10/1731259344666/Put/seqid=0 2024-11-10T17:22:25,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741917_1093 (size=12301) 2024-11-10T17:22:25,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/94e2af5f320d4a829cc2e1df6686e580 2024-11-10T17:22:25,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/cfc23839af2741baa5f42c455c8d6006 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cfc23839af2741baa5f42c455c8d6006 2024-11-10T17:22:25,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cfc23839af2741baa5f42c455c8d6006, entries=150, sequenceid=381, filesize=12.0 K 2024-11-10T17:22:25,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/02af42dd510f43ac8ca32a9fef0a62dc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/02af42dd510f43ac8ca32a9fef0a62dc 2024-11-10T17:22:25,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/02af42dd510f43ac8ca32a9fef0a62dc, entries=150, sequenceid=381, filesize=12.0 K 2024-11-10T17:22:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/94e2af5f320d4a829cc2e1df6686e580 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/94e2af5f320d4a829cc2e1df6686e580 2024-11-10T17:22:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T17:22:25,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T17:22:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:25,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:25,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/94e2af5f320d4a829cc2e1df6686e580, entries=150, sequenceid=381, filesize=12.0 K 2024-11-10T17:22:25,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 86036b2e5b1a6747cefd43fa6b653d49 in 543ms, sequenceid=381, compaction requested=false 2024-11-10T17:22:25,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:25,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T17:22:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:25,348 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:22:25,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/7e4e31b7fade463ea82260d9ea37fa2e is 50, key is test_row_0/A:col10/1731259344759/Put/seqid=0 2024-11-10T17:22:25,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741918_1094 (size=12301) 2024-11-10T17:22:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:25,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:25,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259405401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259405404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259405410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T17:22:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259405512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259405512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259405521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259405722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259405722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259405732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,789 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/7e4e31b7fade463ea82260d9ea37fa2e 2024-11-10T17:22:25,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259405791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,795 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 86036b2e5b1a6747cefd43fa6b653d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/bcf9983b2afd4be78d4efbb8abae33e9 is 50, key is test_row_0/B:col10/1731259344759/Put/seqid=0 2024-11-10T17:22:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259405812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:25,813 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 86036b2e5b1a6747cefd43fa6b653d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:25,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741919_1095 (size=12301) 2024-11-10T17:22:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T17:22:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259406032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259406032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259406044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,223 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/bcf9983b2afd4be78d4efbb8abae33e9 2024-11-10T17:22:26,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/386a465033cc40e2a6bf8ae3564a3ef5 is 50, key is test_row_0/C:col10/1731259344759/Put/seqid=0 2024-11-10T17:22:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741920_1096 (size=12301) 2024-11-10T17:22:26,255 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/386a465033cc40e2a6bf8ae3564a3ef5 2024-11-10T17:22:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/7e4e31b7fade463ea82260d9ea37fa2e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/7e4e31b7fade463ea82260d9ea37fa2e 2024-11-10T17:22:26,287 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/7e4e31b7fade463ea82260d9ea37fa2e, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T17:22:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/bcf9983b2afd4be78d4efbb8abae33e9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/bcf9983b2afd4be78d4efbb8abae33e9 2024-11-10T17:22:26,295 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/bcf9983b2afd4be78d4efbb8abae33e9, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T17:22:26,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/386a465033cc40e2a6bf8ae3564a3ef5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/386a465033cc40e2a6bf8ae3564a3ef5 2024-11-10T17:22:26,306 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/386a465033cc40e2a6bf8ae3564a3ef5, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T17:22:26,307 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 86036b2e5b1a6747cefd43fa6b653d49 in 960ms, sequenceid=408, compaction requested=true 2024-11-10T17:22:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T17:22:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T17:22:26,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T17:22:26,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-11-10T17:22:26,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4420 sec 2024-11-10T17:22:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:26,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:26,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/ba0c573322cc4add9f8db49373b88735 is 50, key is test_row_0/A:col10/1731259346544/Put/seqid=0 2024-11-10T17:22:26,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741921_1097 (size=12301) 2024-11-10T17:22:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259406601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259406602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259406606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259406711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259406712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259406712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259406922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259406923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259406923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:26,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/ba0c573322cc4add9f8db49373b88735 2024-11-10T17:22:26,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/9d654446f34645c6a4500eee22979a68 is 50, key is test_row_0/B:col10/1731259346544/Put/seqid=0 2024-11-10T17:22:27,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741922_1098 (size=12301) 2024-11-10T17:22:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T17:22:27,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/9d654446f34645c6a4500eee22979a68 2024-11-10T17:22:27,023 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:27,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T17:22:27,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:27,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:27,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T17:22:27,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/f42b569ee35c462d827c54a832caf268 is 50, key is test_row_0/C:col10/1731259346544/Put/seqid=0 2024-11-10T17:22:27,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741923_1099 (size=12301) 2024-11-10T17:22:27,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/f42b569ee35c462d827c54a832caf268 2024-11-10T17:22:27,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/ba0c573322cc4add9f8db49373b88735 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ba0c573322cc4add9f8db49373b88735 2024-11-10T17:22:27,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ba0c573322cc4add9f8db49373b88735, entries=150, sequenceid=420, filesize=12.0 K 2024-11-10T17:22:27,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/9d654446f34645c6a4500eee22979a68 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9d654446f34645c6a4500eee22979a68 2024-11-10T17:22:27,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9d654446f34645c6a4500eee22979a68, entries=150, sequenceid=420, filesize=12.0 K 2024-11-10T17:22:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/f42b569ee35c462d827c54a832caf268 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f42b569ee35c462d827c54a832caf268 2024-11-10T17:22:27,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f42b569ee35c462d827c54a832caf268, entries=150, sequenceid=420, filesize=12.0 K 2024-11-10T17:22:27,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 86036b2e5b1a6747cefd43fa6b653d49 in 556ms, sequenceid=420, compaction requested=true 2024-11-10T17:22:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:27,101 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:27,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:27,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:27,102 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:27,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:27,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:27,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:27,103 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:27,103 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:27,103 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:27,103 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/23ec77af2e1642268d46ee53c8fa908b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cfc23839af2741baa5f42c455c8d6006, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/7e4e31b7fade463ea82260d9ea37fa2e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ba0c573322cc4add9f8db49373b88735] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=48.9 K 2024-11-10T17:22:27,104 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ec77af2e1642268d46ee53c8fa908b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731259342480 2024-11-10T17:22:27,104 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:27,104 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc23839af2741baa5f42c455c8d6006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731259344664 2024-11-10T17:22:27,104 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:27,104 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:27,105 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/e27aaf631542450aa57fd549824ec220, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/02af42dd510f43ac8ca32a9fef0a62dc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/bcf9983b2afd4be78d4efbb8abae33e9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9d654446f34645c6a4500eee22979a68] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=48.9 K 2024-11-10T17:22:27,105 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e4e31b7fade463ea82260d9ea37fa2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731259344753 2024-11-10T17:22:27,105 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e27aaf631542450aa57fd549824ec220, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731259342480 2024-11-10T17:22:27,105 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0c573322cc4add9f8db49373b88735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731259345394 2024-11-10T17:22:27,106 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 02af42dd510f43ac8ca32a9fef0a62dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731259344664 2024-11-10T17:22:27,106 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf9983b2afd4be78d4efbb8abae33e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731259344753 2024-11-10T17:22:27,107 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d654446f34645c6a4500eee22979a68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731259345394 2024-11-10T17:22:27,127 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:27,128 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/3797ea29d2c54527a00c6e56fe5ad942 is 50, key is test_row_0/B:col10/1731259346544/Put/seqid=0 2024-11-10T17:22:27,141 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T17:22:27,142 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/44b02cfc92d94e3b81a876a3ea7c3f1c is 50, key is test_row_0/A:col10/1731259346544/Put/seqid=0 2024-11-10T17:22:27,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741924_1100 (size=13289) 2024-11-10T17:22:27,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741925_1101 (size=13289) 2024-11-10T17:22:27,162 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/44b02cfc92d94e3b81a876a3ea7c3f1c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/44b02cfc92d94e3b81a876a3ea7c3f1c 2024-11-10T17:22:27,171 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into 44b02cfc92d94e3b81a876a3ea7c3f1c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:27,171 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:27,171 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=12, startTime=1731259347101; duration=0sec 2024-11-10T17:22:27,172 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:27,172 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:27,172 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:27,173 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:27,174 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:27,174 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:27,174 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/979e0e6c020542b697eb22736a5f89a2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/94e2af5f320d4a829cc2e1df6686e580, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/386a465033cc40e2a6bf8ae3564a3ef5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f42b569ee35c462d827c54a832caf268] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=48.9 K 2024-11-10T17:22:27,175 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979e0e6c020542b697eb22736a5f89a2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731259342480 2024-11-10T17:22:27,175 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94e2af5f320d4a829cc2e1df6686e580, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731259344664 2024-11-10T17:22:27,176 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 386a465033cc40e2a6bf8ae3564a3ef5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731259344753 2024-11-10T17:22:27,176 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting f42b569ee35c462d827c54a832caf268, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731259345394 2024-11-10T17:22:27,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T17:22:27,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:27,185 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:22:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/717110e279c54d328e363d29bcca2f32 is 50, key is test_row_0/A:col10/1731259346591/Put/seqid=0 2024-11-10T17:22:27,200 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:27,200 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/9818d126d87347b3af20280e2d8aaaf8 is 50, key is test_row_0/C:col10/1731259346544/Put/seqid=0 2024-11-10T17:22:27,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741926_1102 (size=12301) 2024-11-10T17:22:27,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741927_1103 (size=13289) 2024-11-10T17:22:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:27,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259407255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259407255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259407257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T17:22:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259407361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259407372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259407372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,551 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/3797ea29d2c54527a00c6e56fe5ad942 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3797ea29d2c54527a00c6e56fe5ad942 2024-11-10T17:22:27,558 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 3797ea29d2c54527a00c6e56fe5ad942(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:27,558 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:27,558 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=12, startTime=1731259347102; duration=0sec 2024-11-10T17:22:27,558 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:27,558 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259407572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259407591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259407591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,605 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/717110e279c54d328e363d29bcca2f32 2024-11-10T17:22:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/96025c7865664a53a4fb26373502b160 is 50, key is test_row_0/B:col10/1731259346591/Put/seqid=0 2024-11-10T17:22:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741928_1104 (size=12301) 2024-11-10T17:22:27,631 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/96025c7865664a53a4fb26373502b160 2024-11-10T17:22:27,642 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/9818d126d87347b3af20280e2d8aaaf8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/9818d126d87347b3af20280e2d8aaaf8 2024-11-10T17:22:27,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4c17c0435b724f24a9db3d3afc52b9c9 is 50, key is test_row_0/C:col10/1731259346591/Put/seqid=0 2024-11-10T17:22:27,650 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into 9818d126d87347b3af20280e2d8aaaf8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:27,650 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:27,650 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=12, startTime=1731259347102; duration=0sec 2024-11-10T17:22:27,650 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:27,650 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T17:22:27,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741929_1105 (size=12301) 2024-11-10T17:22:27,667 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4c17c0435b724f24a9db3d3afc52b9c9 2024-11-10T17:22:27,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/717110e279c54d328e363d29bcca2f32 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/717110e279c54d328e363d29bcca2f32 2024-11-10T17:22:27,680 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/717110e279c54d328e363d29bcca2f32, entries=150, sequenceid=445, filesize=12.0 K 2024-11-10T17:22:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/96025c7865664a53a4fb26373502b160 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/96025c7865664a53a4fb26373502b160 2024-11-10T17:22:27,689 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/96025c7865664a53a4fb26373502b160, entries=150, sequenceid=445, filesize=12.0 K 2024-11-10T17:22:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4c17c0435b724f24a9db3d3afc52b9c9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4c17c0435b724f24a9db3d3afc52b9c9 2024-11-10T17:22:27,695 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4c17c0435b724f24a9db3d3afc52b9c9, entries=150, sequenceid=445, filesize=12.0 K 2024-11-10T17:22:27,697 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 86036b2e5b1a6747cefd43fa6b653d49 in 511ms, sequenceid=445, compaction requested=false 2024-11-10T17:22:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T17:22:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T17:22:27,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T17:22:27,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 668 msec 2024-11-10T17:22:27,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 678 msec 2024-11-10T17:22:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:27,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:27,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:22:27,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/c9718cb050334047ae571b7c4ee27ddc is 50, key is test_row_0/A:col10/1731259347240/Put/seqid=0 2024-11-10T17:22:27,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741930_1106 (size=14741) 2024-11-10T17:22:27,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/c9718cb050334047ae571b7c4ee27ddc 2024-11-10T17:22:27,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5d794bfda8914bf08d579b4266f74abc is 50, key is test_row_0/B:col10/1731259347240/Put/seqid=0 2024-11-10T17:22:27,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741931_1107 (size=12301) 2024-11-10T17:22:27,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5d794bfda8914bf08d579b4266f74abc 2024-11-10T17:22:27,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259407892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259407892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259407892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/6f381db1911149eb8d018ff2d5b21992 is 50, key is test_row_0/C:col10/1731259347240/Put/seqid=0 2024-11-10T17:22:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259407901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259407901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:27,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:27,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741932_1108 (size=12301) 2024-11-10T17:22:28,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259408001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259408002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259408002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T17:22:28,172 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:28,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T17:22:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T17:22:28,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:28,176 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:28,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:28,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259408212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259408212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259408212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T17:22:28,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/6f381db1911149eb8d018ff2d5b21992 2024-11-10T17:22:28,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/c9718cb050334047ae571b7c4ee27ddc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/c9718cb050334047ae571b7c4ee27ddc 2024-11-10T17:22:28,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/c9718cb050334047ae571b7c4ee27ddc, entries=200, sequenceid=460, filesize=14.4 K 2024-11-10T17:22:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5d794bfda8914bf08d579b4266f74abc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5d794bfda8914bf08d579b4266f74abc 2024-11-10T17:22:28,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5d794bfda8914bf08d579b4266f74abc, entries=150, sequenceid=460, filesize=12.0 K 2024-11-10T17:22:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/6f381db1911149eb8d018ff2d5b21992 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6f381db1911149eb8d018ff2d5b21992 2024-11-10T17:22:28,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6f381db1911149eb8d018ff2d5b21992, entries=150, sequenceid=460, filesize=12.0 K 2024-11-10T17:22:28,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 86036b2e5b1a6747cefd43fa6b653d49 in 534ms, sequenceid=460, compaction requested=true 2024-11-10T17:22:28,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:28,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:28,358 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:28,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:28,359 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:28,362 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:28,362 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:28,362 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:28,362 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:28,362 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,363 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,363 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/44b02cfc92d94e3b81a876a3ea7c3f1c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/717110e279c54d328e363d29bcca2f32, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/c9718cb050334047ae571b7c4ee27ddc] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=39.4 K 2024-11-10T17:22:28,363 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3797ea29d2c54527a00c6e56fe5ad942, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/96025c7865664a53a4fb26373502b160, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5d794bfda8914bf08d579b4266f74abc] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=37.0 K 2024-11-10T17:22:28,363 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44b02cfc92d94e3b81a876a3ea7c3f1c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731259345394 2024-11-10T17:22:28,363 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3797ea29d2c54527a00c6e56fe5ad942, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731259345394 2024-11-10T17:22:28,364 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 96025c7865664a53a4fb26373502b160, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731259346591 2024-11-10T17:22:28,364 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717110e279c54d328e363d29bcca2f32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731259346591 2024-11-10T17:22:28,364 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d794bfda8914bf08d579b4266f74abc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731259347240 2024-11-10T17:22:28,365 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9718cb050334047ae571b7c4ee27ddc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731259347240 2024-11-10T17:22:28,378 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:28,380 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/8fcb70d86c2a42dfabd9cc06de98df52 is 50, key is test_row_0/A:col10/1731259347240/Put/seqid=0 2024-11-10T17:22:28,397 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#97 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:28,398 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7461123c41d04165b8e9d0f5242befa3 is 50, key is test_row_0/B:col10/1731259347240/Put/seqid=0 2024-11-10T17:22:28,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741933_1109 (size=13391) 2024-11-10T17:22:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:28,416 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:28,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:22:28,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741934_1110 (size=13391) 2024-11-10T17:22:28,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3db01b655a914236b43a45d06e7e6bc8 is 50, key is test_row_0/A:col10/1731259348414/Put/seqid=0 2024-11-10T17:22:28,446 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7461123c41d04165b8e9d0f5242befa3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7461123c41d04165b8e9d0f5242befa3 2024-11-10T17:22:28,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259408445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259408447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,454 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 7461123c41d04165b8e9d0f5242befa3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:28,454 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:28,454 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259348358; duration=0sec 2024-11-10T17:22:28,455 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:28,455 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:28,455 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:28,456 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:28,456 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:28,457 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,457 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/9818d126d87347b3af20280e2d8aaaf8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4c17c0435b724f24a9db3d3afc52b9c9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6f381db1911149eb8d018ff2d5b21992] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=37.0 K 2024-11-10T17:22:28,458 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 9818d126d87347b3af20280e2d8aaaf8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731259345394 2024-11-10T17:22:28,458 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c17c0435b724f24a9db3d3afc52b9c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731259346591 2024-11-10T17:22:28,459 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f381db1911149eb8d018ff2d5b21992, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731259347240 2024-11-10T17:22:28,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741935_1111 (size=14741) 2024-11-10T17:22:28,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3db01b655a914236b43a45d06e7e6bc8 2024-11-10T17:22:28,482 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:28,483 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/f34c08faf4b749a6ba43b842a78c6bb4 is 50, key is test_row_0/C:col10/1731259347240/Put/seqid=0 2024-11-10T17:22:28,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/d6a36d6e8e2e419e86af7032bbc5d4ce is 50, key is test_row_0/B:col10/1731259348414/Put/seqid=0 2024-11-10T17:22:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T17:22:28,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741936_1112 (size=13391) 2024-11-10T17:22:28,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741937_1113 (size=12301) 2024-11-10T17:22:28,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259408522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259408522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259408522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259408551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259408552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:28,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259408761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259408762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T17:22:28,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,811 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/8fcb70d86c2a42dfabd9cc06de98df52 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8fcb70d86c2a42dfabd9cc06de98df52 2024-11-10T17:22:28,822 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into 8fcb70d86c2a42dfabd9cc06de98df52(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:28,822 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:28,822 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259348358; duration=0sec 2024-11-10T17:22:28,822 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:28,822 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:28,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/d6a36d6e8e2e419e86af7032bbc5d4ce 2024-11-10T17:22:28,928 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/f34c08faf4b749a6ba43b842a78c6bb4 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f34c08faf4b749a6ba43b842a78c6bb4 2024-11-10T17:22:28,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/661bb3e3b0954ce893f67f99fd69eb8e is 50, key is test_row_0/C:col10/1731259348414/Put/seqid=0 2024-11-10T17:22:28,939 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into f34c08faf4b749a6ba43b842a78c6bb4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:28,939 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:28,940 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259348359; duration=0sec 2024-11-10T17:22:28,940 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:28,940 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:28,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741938_1114 (size=12301) 2024-11-10T17:22:28,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/661bb3e3b0954ce893f67f99fd69eb8e 2024-11-10T17:22:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/3db01b655a914236b43a45d06e7e6bc8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3db01b655a914236b43a45d06e7e6bc8 2024-11-10T17:22:28,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:28,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:28,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3db01b655a914236b43a45d06e7e6bc8, entries=200, sequenceid=484, filesize=14.4 K 2024-11-10T17:22:28,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/d6a36d6e8e2e419e86af7032bbc5d4ce as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d6a36d6e8e2e419e86af7032bbc5d4ce 2024-11-10T17:22:28,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d6a36d6e8e2e419e86af7032bbc5d4ce, entries=150, sequenceid=484, filesize=12.0 K 2024-11-10T17:22:28,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/661bb3e3b0954ce893f67f99fd69eb8e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/661bb3e3b0954ce893f67f99fd69eb8e 2024-11-10T17:22:28,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/661bb3e3b0954ce893f67f99fd69eb8e, entries=150, sequenceid=484, filesize=12.0 K 2024-11-10T17:22:28,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 86036b2e5b1a6747cefd43fa6b653d49 in 570ms, sequenceid=484, compaction requested=false 2024-11-10T17:22:28,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:29,035 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:29,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T17:22:29,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/b7620df90bac4c0b82c6eed7253cad60 is 50, key is test_row_0/A:col10/1731259349035/Put/seqid=0 2024-11-10T17:22:29,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741939_1115 (size=14741) 2024-11-10T17:22:29,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/b7620df90bac4c0b82c6eed7253cad60 2024-11-10T17:22:29,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7665ee74792f4092a9592ea06bf7c178 is 50, key is test_row_0/B:col10/1731259349035/Put/seqid=0 2024-11-10T17:22:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259409090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259409091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259409091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259409099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259409102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741940_1116 (size=12301) 2024-11-10T17:22:29,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7665ee74792f4092a9592ea06bf7c178 2024-11-10T17:22:29,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/b1f892c0f27c44e2a873df8f68ea0ce1 is 50, key is test_row_0/C:col10/1731259349035/Put/seqid=0 2024-11-10T17:22:29,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741941_1117 (size=12301) 2024-11-10T17:22:29,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/b1f892c0f27c44e2a873df8f68ea0ce1 2024-11-10T17:22:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/b7620df90bac4c0b82c6eed7253cad60 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/b7620df90bac4c0b82c6eed7253cad60 2024-11-10T17:22:29,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/b7620df90bac4c0b82c6eed7253cad60, entries=200, sequenceid=502, filesize=14.4 K 2024-11-10T17:22:29,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/7665ee74792f4092a9592ea06bf7c178 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7665ee74792f4092a9592ea06bf7c178 2024-11-10T17:22:29,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7665ee74792f4092a9592ea06bf7c178, entries=150, sequenceid=502, filesize=12.0 K 2024-11-10T17:22:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/b1f892c0f27c44e2a873df8f68ea0ce1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/b1f892c0f27c44e2a873df8f68ea0ce1 2024-11-10T17:22:29,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259409204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259409204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/b1f892c0f27c44e2a873df8f68ea0ce1, entries=150, sequenceid=502, filesize=12.0 K 2024-11-10T17:22:29,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 86036b2e5b1a6747cefd43fa6b653d49 in 179ms, sequenceid=502, compaction requested=true 2024-11-10T17:22:29,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:29,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:29,214 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:29,215 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:29,217 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:29,217 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:29,217 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:29,217 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:29,217 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,217 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,217 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7461123c41d04165b8e9d0f5242befa3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d6a36d6e8e2e419e86af7032bbc5d4ce, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7665ee74792f4092a9592ea06bf7c178] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=37.1 K 2024-11-10T17:22:29,218 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7461123c41d04165b8e9d0f5242befa3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731259347240 2024-11-10T17:22:29,218 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8fcb70d86c2a42dfabd9cc06de98df52, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3db01b655a914236b43a45d06e7e6bc8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/b7620df90bac4c0b82c6eed7253cad60] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=41.9 K 2024-11-10T17:22:29,219 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fcb70d86c2a42dfabd9cc06de98df52, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731259347240 2024-11-10T17:22:29,219 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a36d6e8e2e419e86af7032bbc5d4ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731259347849 2024-11-10T17:22:29,220 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3db01b655a914236b43a45d06e7e6bc8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731259347849 2024-11-10T17:22:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:29,221 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7665ee74792f4092a9592ea06bf7c178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731259348434 2024-11-10T17:22:29,221 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7620df90bac4c0b82c6eed7253cad60, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731259348434 2024-11-10T17:22:29,225 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:29,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:22:29,233 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:29,234 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/20947b135d0f474ba14ef76ed667569b is 50, key is test_row_0/B:col10/1731259349035/Put/seqid=0 2024-11-10T17:22:29,239 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:29,240 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/df8dd84f741c4720bc5f99714d869c4e is 50, key is test_row_0/A:col10/1731259349035/Put/seqid=0 2024-11-10T17:22:29,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/88baa43a22a3413a95424419122a3cc9 is 50, key is test_row_0/A:col10/1731259349095/Put/seqid=0 2024-11-10T17:22:29,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741942_1118 (size=13493) 2024-11-10T17:22:29,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259409256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259409257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741943_1119 (size=13493) 2024-11-10T17:22:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259409261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741944_1120 (size=12301) 2024-11-10T17:22:29,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/88baa43a22a3413a95424419122a3cc9 2024-11-10T17:22:29,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:29,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:29,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c8ee725444334d56a303f0f8a940188f is 50, key is test_row_0/B:col10/1731259349095/Put/seqid=0 2024-11-10T17:22:29,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741945_1121 (size=12301) 2024-11-10T17:22:29,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c8ee725444334d56a303f0f8a940188f 2024-11-10T17:22:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T17:22:29,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/041f7be730ed49bdad97f9776860b985 is 50, key is test_row_0/C:col10/1731259349095/Put/seqid=0 2024-11-10T17:22:29,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741946_1122 (size=12301) 2024-11-10T17:22:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259409372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259409372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259409372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259409412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259409422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:29,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:29,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259409582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259409582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259409582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,662 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/20947b135d0f474ba14ef76ed667569b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/20947b135d0f474ba14ef76ed667569b 2024-11-10T17:22:29,674 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 20947b135d0f474ba14ef76ed667569b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:29,674 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:29,674 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259349215; duration=0sec 2024-11-10T17:22:29,674 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:29,674 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:29,674 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:29,675 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/df8dd84f741c4720bc5f99714d869c4e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/df8dd84f741c4720bc5f99714d869c4e 2024-11-10T17:22:29,676 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:29,676 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:29,676 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,677 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f34c08faf4b749a6ba43b842a78c6bb4, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/661bb3e3b0954ce893f67f99fd69eb8e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/b1f892c0f27c44e2a873df8f68ea0ce1] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=37.1 K 2024-11-10T17:22:29,678 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f34c08faf4b749a6ba43b842a78c6bb4, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731259347240 2024-11-10T17:22:29,678 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 661bb3e3b0954ce893f67f99fd69eb8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731259347849 2024-11-10T17:22:29,679 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f892c0f27c44e2a873df8f68ea0ce1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731259348434 2024-11-10T17:22:29,684 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into df8dd84f741c4720bc5f99714d869c4e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:29,684 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:29,684 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259349214; duration=0sec 2024-11-10T17:22:29,684 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:29,684 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:29,694 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:29,695 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/f9c4024de03f4d9683d95deb435b138b is 50, key is test_row_0/C:col10/1731259349035/Put/seqid=0 2024-11-10T17:22:29,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741947_1123 (size=13493) 2024-11-10T17:22:29,706 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/f9c4024de03f4d9683d95deb435b138b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f9c4024de03f4d9683d95deb435b138b 2024-11-10T17:22:29,715 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into f9c4024de03f4d9683d95deb435b138b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:29,715 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:29,715 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259349215; duration=0sec 2024-11-10T17:22:29,715 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:29,715 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259409722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259409731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/041f7be730ed49bdad97f9776860b985 2024-11-10T17:22:29,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/88baa43a22a3413a95424419122a3cc9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/88baa43a22a3413a95424419122a3cc9 2024-11-10T17:22:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/88baa43a22a3413a95424419122a3cc9, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T17:22:29,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c8ee725444334d56a303f0f8a940188f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c8ee725444334d56a303f0f8a940188f 2024-11-10T17:22:29,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c8ee725444334d56a303f0f8a940188f, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T17:22:29,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/041f7be730ed49bdad97f9776860b985 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/041f7be730ed49bdad97f9776860b985 2024-11-10T17:22:29,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/041f7be730ed49bdad97f9776860b985, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T17:22:29,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 86036b2e5b1a6747cefd43fa6b653d49 in 554ms, sequenceid=527, compaction requested=false 2024-11-10T17:22:29,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:29,894 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:29,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:22:29,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:29,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:29,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/531956a277dc485d94f976bab56ff295 is 50, key is test_row_0/A:col10/1731259349893/Put/seqid=0 2024-11-10T17:22:29,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741948_1124 (size=17181) 2024-11-10T17:22:29,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259409941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259409942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259409944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259410052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259410062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259410062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:30,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259410231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259410242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259410262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259410271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259410281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T17:22:30,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/531956a277dc485d94f976bab56ff295 2024-11-10T17:22:30,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/555b6a8535ea4414a165e9ca746cc877 is 50, key is test_row_0/B:col10/1731259349893/Put/seqid=0 2024-11-10T17:22:30,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741949_1125 (size=12301) 2024-11-10T17:22:30,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/555b6a8535ea4414a165e9ca746cc877 2024-11-10T17:22:30,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:30,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:30,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4488ce220e7a419384a41c33792fb2cb is 50, key is test_row_0/C:col10/1731259349893/Put/seqid=0 2024-11-10T17:22:30,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741950_1126 (size=12301) 2024-11-10T17:22:30,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259410571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259410581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259410592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:30,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4488ce220e7a419384a41c33792fb2cb 2024-11-10T17:22:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/531956a277dc485d94f976bab56ff295 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/531956a277dc485d94f976bab56ff295 2024-11-10T17:22:30,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/531956a277dc485d94f976bab56ff295, entries=250, sequenceid=543, filesize=16.8 K 2024-11-10T17:22:30,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/555b6a8535ea4414a165e9ca746cc877 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/555b6a8535ea4414a165e9ca746cc877 2024-11-10T17:22:30,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/555b6a8535ea4414a165e9ca746cc877, entries=150, sequenceid=543, filesize=12.0 K 2024-11-10T17:22:30,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/4488ce220e7a419384a41c33792fb2cb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4488ce220e7a419384a41c33792fb2cb 2024-11-10T17:22:30,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:30,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4488ce220e7a419384a41c33792fb2cb, entries=150, sequenceid=543, filesize=12.0 K 2024-11-10T17:22:30,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 86036b2e5b1a6747cefd43fa6b653d49 in 930ms, sequenceid=543, compaction requested=true 2024-11-10T17:22:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:30,825 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:30,825 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:30,827 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:30,827 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/A is initiating minor compaction (all files) 2024-11-10T17:22:30,827 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/A in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,827 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/df8dd84f741c4720bc5f99714d869c4e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/88baa43a22a3413a95424419122a3cc9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/531956a277dc485d94f976bab56ff295] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=42.0 K 2024-11-10T17:22:30,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:30,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:30,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:30,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:30,828 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:30,828 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/B is initiating minor compaction (all files) 2024-11-10T17:22:30,828 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/B in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,829 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/20947b135d0f474ba14ef76ed667569b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c8ee725444334d56a303f0f8a940188f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/555b6a8535ea4414a165e9ca746cc877] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=37.2 K 2024-11-10T17:22:30,829 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting df8dd84f741c4720bc5f99714d869c4e, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731259348434 2024-11-10T17:22:30,830 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 20947b135d0f474ba14ef76ed667569b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731259348434 2024-11-10T17:22:30,830 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88baa43a22a3413a95424419122a3cc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731259349095 2024-11-10T17:22:30,831 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ee725444334d56a303f0f8a940188f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731259349095 2024-11-10T17:22:30,831 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 531956a277dc485d94f976bab56ff295, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731259349243 2024-11-10T17:22:30,831 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 555b6a8535ea4414a165e9ca746cc877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731259349253 2024-11-10T17:22:30,859 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#A#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:30,859 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/8c62dc47290e4203b1bf461f70ffb15c is 50, key is test_row_0/A:col10/1731259349893/Put/seqid=0 2024-11-10T17:22:30,861 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:30,862 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5286040e3f53483daa16d56650a6df0a is 50, key is test_row_0/B:col10/1731259349893/Put/seqid=0 2024-11-10T17:22:30,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741952_1128 (size=13595) 2024-11-10T17:22:30,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741951_1127 (size=13595) 2024-11-10T17:22:30,893 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/5286040e3f53483daa16d56650a6df0a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5286040e3f53483daa16d56650a6df0a 2024-11-10T17:22:30,898 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/8c62dc47290e4203b1bf461f70ffb15c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8c62dc47290e4203b1bf461f70ffb15c 2024-11-10T17:22:30,901 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/B of 86036b2e5b1a6747cefd43fa6b653d49 into 5286040e3f53483daa16d56650a6df0a(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:30,901 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:30,901 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/B, priority=13, startTime=1731259350825; duration=0sec 2024-11-10T17:22:30,901 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:30,901 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:30,901 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:30,903 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:30,903 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 86036b2e5b1a6747cefd43fa6b653d49/C is initiating minor compaction (all files) 2024-11-10T17:22:30,903 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86036b2e5b1a6747cefd43fa6b653d49/C in TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,903 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f9c4024de03f4d9683d95deb435b138b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/041f7be730ed49bdad97f9776860b985, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4488ce220e7a419384a41c33792fb2cb] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp, totalSize=37.2 K 2024-11-10T17:22:30,904 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c4024de03f4d9683d95deb435b138b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731259348434 2024-11-10T17:22:30,904 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 041f7be730ed49bdad97f9776860b985, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731259349095 2024-11-10T17:22:30,905 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4488ce220e7a419384a41c33792fb2cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731259349253 2024-11-10T17:22:30,907 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/A of 86036b2e5b1a6747cefd43fa6b653d49 into 8c62dc47290e4203b1bf461f70ffb15c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:30,907 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:30,907 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/A, priority=13, startTime=1731259350825; duration=0sec 2024-11-10T17:22:30,907 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:30,907 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:30,920 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86036b2e5b1a6747cefd43fa6b653d49#C#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:30,920 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/47bb9be5e1ae48f0922a147a69e5b2a1 is 50, key is test_row_0/C:col10/1731259349893/Put/seqid=0 2024-11-10T17:22:30,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741953_1129 (size=13595) 2024-11-10T17:22:30,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T17:22:30,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:30,972 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T17:22:30,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/086545f2f0ee468aa1c145a669d42097 is 50, key is test_row_0/A:col10/1731259349942/Put/seqid=0 2024-11-10T17:22:30,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741954_1130 (size=12301) 2024-11-10T17:22:30,997 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/086545f2f0ee468aa1c145a669d42097 2024-11-10T17:22:31,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/b415870d30324c1f9dcd91927dbb725f is 50, key is test_row_0/B:col10/1731259349942/Put/seqid=0 2024-11-10T17:22:31,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741955_1131 (size=12301) 2024-11-10T17:22:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:31,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. as already flushing 2024-11-10T17:22:31,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259411107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259411114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259411114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259411221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259411221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259411221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56932 deadline: 1731259411252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,255 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 86036b2e5b1a6747cefd43fa6b653d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56888 deadline: 1731259411261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,268 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in 86036b2e5b1a6747cefd43fa6b653d49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,337 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/47bb9be5e1ae48f0922a147a69e5b2a1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/47bb9be5e1ae48f0922a147a69e5b2a1 2024-11-10T17:22:31,350 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86036b2e5b1a6747cefd43fa6b653d49/C of 86036b2e5b1a6747cefd43fa6b653d49 into 47bb9be5e1ae48f0922a147a69e5b2a1(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:31,351 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:31,351 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., storeName=86036b2e5b1a6747cefd43fa6b653d49/C, priority=13, startTime=1731259350827; duration=0sec 2024-11-10T17:22:31,351 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:31,351 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:31,433 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/b415870d30324c1f9dcd91927dbb725f 2024-11-10T17:22:31,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259411432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259411432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259411432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/a989667ec51a44da9251baf5095aadda is 50, key is test_row_0/C:col10/1731259349942/Put/seqid=0 2024-11-10T17:22:31,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741956_1132 (size=12301) 2024-11-10T17:22:31,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56952 deadline: 1731259411742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56938 deadline: 1731259411752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56922 deadline: 1731259411753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:22:31,855 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/a989667ec51a44da9251baf5095aadda 2024-11-10T17:22:31,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/086545f2f0ee468aa1c145a669d42097 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/086545f2f0ee468aa1c145a669d42097 2024-11-10T17:22:31,870 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/086545f2f0ee468aa1c145a669d42097, entries=150, sequenceid=568, filesize=12.0 K 2024-11-10T17:22:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/b415870d30324c1f9dcd91927dbb725f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b415870d30324c1f9dcd91927dbb725f 2024-11-10T17:22:31,878 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b415870d30324c1f9dcd91927dbb725f, entries=150, sequenceid=568, filesize=12.0 K 2024-11-10T17:22:31,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/a989667ec51a44da9251baf5095aadda as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a989667ec51a44da9251baf5095aadda 2024-11-10T17:22:31,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a989667ec51a44da9251baf5095aadda, entries=150, sequenceid=568, filesize=12.0 K 2024-11-10T17:22:31,892 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 86036b2e5b1a6747cefd43fa6b653d49 in 920ms, sequenceid=568, compaction requested=false 2024-11-10T17:22:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T17:22:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T17:22:31,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T17:22:31,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7190 sec 2024-11-10T17:22:31,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 3.7270 sec 2024-11-10T17:22:32,226 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T17:22:32,226 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T17:22:32,226 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:32,226 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:32,226 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T17:22:32,226 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:32,226 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T17:22:32,226 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:32,231 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,231 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,231 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,231 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,231 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:32,232 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:32,232 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:32,232 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:32,254 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:32,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T17:22:32,258 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T17:22:32,258 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:32,258 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,258 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:32,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/621add3028684f5bba35c77bafc49ed5 is 50, key is test_row_0/A:col10/1731259351106/Put/seqid=0 2024-11-10T17:22:32,264 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T17:22:32,264 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T17:22:32,264 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:32,264 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:32,264 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,264 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,264 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,264 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:32,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:32,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:32,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741957_1133 (size=12301) 2024-11-10T17:22:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T17:22:32,353 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:32,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/621add3028684f5bba35c77bafc49ed5 2024-11-10T17:22:32,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c3c9acc9573c47c6b9ec6f0a710b27ac is 50, key is test_row_0/B:col10/1731259351106/Put/seqid=0 2024-11-10T17:22:32,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741958_1134 (size=12301) 2024-11-10T17:22:33,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c3c9acc9573c47c6b9ec6f0a710b27ac 2024-11-10T17:22:33,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/82f9c7e1eb8e49649baae98c819b86fb is 50, key is test_row_0/C:col10/1731259351106/Put/seqid=0 2024-11-10T17:22:33,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741959_1135 (size=12301) 2024-11-10T17:22:33,274 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T17:22:33,274 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:33,274 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:33,274 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:33,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:33,293 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T17:22:33,293 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:33,293 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:33,293 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:33,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2794 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2886 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1315 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3931 rows 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1318 2024-11-10T17:22:33,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3940 rows 2024-11-10T17:22:33,294 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:22:33,294 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:22:33,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:33,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:33,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:33,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T17:22:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:33,307 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259353307"}]},"ts":"1731259353307"} 2024-11-10T17:22:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T17:22:33,309 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T17:22:33,309 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T17:22:33,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:22:33,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, UNASSIGN}] 2024-11-10T17:22:33,316 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, UNASSIGN 2024-11-10T17:22:33,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=86036b2e5b1a6747cefd43fa6b653d49, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:33,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, UNASSIGN because future has completed 2024-11-10T17:22:33,321 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:22:33,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:22:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T17:22:33,478 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:33,478 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:22:33,479 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 86036b2e5b1a6747cefd43fa6b653d49, disabling compactions & flushes 2024-11-10T17:22:33,479 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:33,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/82f9c7e1eb8e49649baae98c819b86fb 2024-11-10T17:22:33,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/621add3028684f5bba35c77bafc49ed5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/621add3028684f5bba35c77bafc49ed5 2024-11-10T17:22:33,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/621add3028684f5bba35c77bafc49ed5, entries=150, sequenceid=584, filesize=12.0 K 2024-11-10T17:22:33,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/c3c9acc9573c47c6b9ec6f0a710b27ac as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c3c9acc9573c47c6b9ec6f0a710b27ac 2024-11-10T17:22:33,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c3c9acc9573c47c6b9ec6f0a710b27ac, entries=150, sequenceid=584, filesize=12.0 K 2024-11-10T17:22:33,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/82f9c7e1eb8e49649baae98c819b86fb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/82f9c7e1eb8e49649baae98c819b86fb 2024-11-10T17:22:33,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/82f9c7e1eb8e49649baae98c819b86fb, entries=150, sequenceid=584, filesize=12.0 K 2024-11-10T17:22:33,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=26.84 KB/27480 for 86036b2e5b1a6747cefd43fa6b653d49 in 1264ms, sequenceid=584, compaction requested=true 2024-11-10T17:22:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86036b2e5b1a6747cefd43fa6b653d49: 2024-11-10T17:22:33,518 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:33,518 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:33,518 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. after waiting 0 ms 2024-11-10T17:22:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:33,518 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86036b2e5b1a6747cefd43fa6b653d49:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:33,518 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. because compaction request was cancelled 2024-11-10T17:22:33,518 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. because compaction request was cancelled 2024-11-10T17:22:33,518 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:B 2024-11-10T17:22:33,518 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:A 2024-11-10T17:22:33,518 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 86036b2e5b1a6747cefd43fa6b653d49 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T17:22:33,519 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. because compaction request was cancelled 2024-11-10T17:22:33,519 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86036b2e5b1a6747cefd43fa6b653d49:C 2024-11-10T17:22:33,522 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/eeb920e502b740ad833734b3bb5ba933 is 50, key is test_row_0/A:col10/1731259353291/Put/seqid=0 2024-11-10T17:22:33,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741960_1136 (size=12301) 2024-11-10T17:22:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T17:22:33,927 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/eeb920e502b740ad833734b3bb5ba933 2024-11-10T17:22:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T17:22:33,935 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/6eaa1debcea348f8a6c0d24d348fa695 is 50, key is test_row_0/B:col10/1731259353291/Put/seqid=0 2024-11-10T17:22:33,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741961_1137 (size=12301) 2024-11-10T17:22:34,340 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/6eaa1debcea348f8a6c0d24d348fa695 2024-11-10T17:22:34,348 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/90d7f2083a4742d8b8819c6f16bf1739 is 50, key is test_row_0/C:col10/1731259353291/Put/seqid=0 2024-11-10T17:22:34,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741962_1138 (size=12301) 2024-11-10T17:22:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T17:22:34,753 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/90d7f2083a4742d8b8819c6f16bf1739 2024-11-10T17:22:34,758 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/A/eeb920e502b740ad833734b3bb5ba933 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/eeb920e502b740ad833734b3bb5ba933 2024-11-10T17:22:34,762 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/eeb920e502b740ad833734b3bb5ba933, entries=150, sequenceid=591, filesize=12.0 K 2024-11-10T17:22:34,762 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/B/6eaa1debcea348f8a6c0d24d348fa695 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/6eaa1debcea348f8a6c0d24d348fa695 2024-11-10T17:22:34,767 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/6eaa1debcea348f8a6c0d24d348fa695, entries=150, sequenceid=591, filesize=12.0 K 2024-11-10T17:22:34,768 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/.tmp/C/90d7f2083a4742d8b8819c6f16bf1739 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/90d7f2083a4742d8b8819c6f16bf1739 2024-11-10T17:22:34,772 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/90d7f2083a4742d8b8819c6f16bf1739, entries=150, sequenceid=591, filesize=12.0 K 2024-11-10T17:22:34,773 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 86036b2e5b1a6747cefd43fa6b653d49 in 1254ms, sequenceid=591, compaction requested=true 2024-11-10T17:22:34,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/bc18cf810cf54772a7193496816d258c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/63ab4a29ea3945c59552012c1e332568, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4ef6eee48ef3422688760e65537e35d3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/53f9f1e37d9a4176b75f6666697368ba, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/047f11a9977e44a48cbd5b3af96e8741, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3321df543c184d5fbb48a01b33ef6d8b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/6a26b13261e74a98b156b1cdfa1f2c83, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/861ec654e5804f5db3120180ff2099cd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/f105ce5e89194fada59333825b396c5d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/69f55c33687843b4ad9e48480b733218, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/89d2670e3b14482ebde63250230851d9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ebb180bf761c4e3faae2b310d212c91f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/a03dee0eae11411b97292bd118439ef5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3bb27e5037464fad9f97a1dc0e44f000, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/82aa14a628474b7f8eab80bdb32c319d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4515512378b4415bb374b14515fa0240, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/734ed7485a754ae091f53772872209fb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/e44ace6dd4004cb5abfd79b2eb175ca2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/dd5440b93f3c450da5f1684e3a7d9bd8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/47370ae15de744429991368cf1fe1451, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/95c9d14b600e4ab1ac4477677d8f83e6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cb0e8727784549e7a97a1fec044cbf7a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/1f578ce8df7e4caa8e0c2b8128036ffe, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/5a46f8764e904297a66ddf155d52d268, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/23ec77af2e1642268d46ee53c8fa908b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/49ea190fdf70415cb47955d710d0ff22, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cfc23839af2741baa5f42c455c8d6006, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/7e4e31b7fade463ea82260d9ea37fa2e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/44b02cfc92d94e3b81a876a3ea7c3f1c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ba0c573322cc4add9f8db49373b88735, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/717110e279c54d328e363d29bcca2f32, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/c9718cb050334047ae571b7c4ee27ddc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8fcb70d86c2a42dfabd9cc06de98df52, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3db01b655a914236b43a45d06e7e6bc8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/b7620df90bac4c0b82c6eed7253cad60, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/df8dd84f741c4720bc5f99714d869c4e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/88baa43a22a3413a95424419122a3cc9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/531956a277dc485d94f976bab56ff295] to archive 2024-11-10T17:22:34,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:22:34,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/bc18cf810cf54772a7193496816d258c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/bc18cf810cf54772a7193496816d258c 2024-11-10T17:22:34,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/63ab4a29ea3945c59552012c1e332568 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/63ab4a29ea3945c59552012c1e332568 2024-11-10T17:22:34,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4ef6eee48ef3422688760e65537e35d3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4ef6eee48ef3422688760e65537e35d3 2024-11-10T17:22:34,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/53f9f1e37d9a4176b75f6666697368ba to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/53f9f1e37d9a4176b75f6666697368ba 2024-11-10T17:22:34,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/047f11a9977e44a48cbd5b3af96e8741 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/047f11a9977e44a48cbd5b3af96e8741 2024-11-10T17:22:34,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3321df543c184d5fbb48a01b33ef6d8b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3321df543c184d5fbb48a01b33ef6d8b 2024-11-10T17:22:34,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/6a26b13261e74a98b156b1cdfa1f2c83 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/6a26b13261e74a98b156b1cdfa1f2c83 2024-11-10T17:22:34,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/861ec654e5804f5db3120180ff2099cd to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/861ec654e5804f5db3120180ff2099cd 2024-11-10T17:22:34,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/f105ce5e89194fada59333825b396c5d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/f105ce5e89194fada59333825b396c5d 2024-11-10T17:22:34,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/69f55c33687843b4ad9e48480b733218 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/69f55c33687843b4ad9e48480b733218 2024-11-10T17:22:34,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/89d2670e3b14482ebde63250230851d9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/89d2670e3b14482ebde63250230851d9 2024-11-10T17:22:34,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ebb180bf761c4e3faae2b310d212c91f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ebb180bf761c4e3faae2b310d212c91f 2024-11-10T17:22:34,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/a03dee0eae11411b97292bd118439ef5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/a03dee0eae11411b97292bd118439ef5 2024-11-10T17:22:34,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3bb27e5037464fad9f97a1dc0e44f000 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3bb27e5037464fad9f97a1dc0e44f000 2024-11-10T17:22:34,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/82aa14a628474b7f8eab80bdb32c319d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/82aa14a628474b7f8eab80bdb32c319d 2024-11-10T17:22:34,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4515512378b4415bb374b14515fa0240 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/4515512378b4415bb374b14515fa0240 2024-11-10T17:22:34,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/734ed7485a754ae091f53772872209fb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/734ed7485a754ae091f53772872209fb 2024-11-10T17:22:34,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/e44ace6dd4004cb5abfd79b2eb175ca2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/e44ace6dd4004cb5abfd79b2eb175ca2 2024-11-10T17:22:34,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/dd5440b93f3c450da5f1684e3a7d9bd8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/dd5440b93f3c450da5f1684e3a7d9bd8 2024-11-10T17:22:34,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/47370ae15de744429991368cf1fe1451 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/47370ae15de744429991368cf1fe1451 2024-11-10T17:22:34,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/95c9d14b600e4ab1ac4477677d8f83e6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/95c9d14b600e4ab1ac4477677d8f83e6 2024-11-10T17:22:34,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cb0e8727784549e7a97a1fec044cbf7a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cb0e8727784549e7a97a1fec044cbf7a 2024-11-10T17:22:34,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/1f578ce8df7e4caa8e0c2b8128036ffe to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/1f578ce8df7e4caa8e0c2b8128036ffe 2024-11-10T17:22:34,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/5a46f8764e904297a66ddf155d52d268 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/5a46f8764e904297a66ddf155d52d268 2024-11-10T17:22:34,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/23ec77af2e1642268d46ee53c8fa908b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/23ec77af2e1642268d46ee53c8fa908b 2024-11-10T17:22:34,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/49ea190fdf70415cb47955d710d0ff22 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/49ea190fdf70415cb47955d710d0ff22 2024-11-10T17:22:34,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cfc23839af2741baa5f42c455c8d6006 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/cfc23839af2741baa5f42c455c8d6006 2024-11-10T17:22:34,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/7e4e31b7fade463ea82260d9ea37fa2e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/7e4e31b7fade463ea82260d9ea37fa2e 2024-11-10T17:22:34,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/44b02cfc92d94e3b81a876a3ea7c3f1c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/44b02cfc92d94e3b81a876a3ea7c3f1c 2024-11-10T17:22:34,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ba0c573322cc4add9f8db49373b88735 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/ba0c573322cc4add9f8db49373b88735 2024-11-10T17:22:34,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/717110e279c54d328e363d29bcca2f32 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/717110e279c54d328e363d29bcca2f32 2024-11-10T17:22:34,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/c9718cb050334047ae571b7c4ee27ddc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/c9718cb050334047ae571b7c4ee27ddc 2024-11-10T17:22:34,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8fcb70d86c2a42dfabd9cc06de98df52 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8fcb70d86c2a42dfabd9cc06de98df52 2024-11-10T17:22:34,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3db01b655a914236b43a45d06e7e6bc8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/3db01b655a914236b43a45d06e7e6bc8 2024-11-10T17:22:34,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/b7620df90bac4c0b82c6eed7253cad60 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/b7620df90bac4c0b82c6eed7253cad60 2024-11-10T17:22:34,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/df8dd84f741c4720bc5f99714d869c4e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/df8dd84f741c4720bc5f99714d869c4e 2024-11-10T17:22:34,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/88baa43a22a3413a95424419122a3cc9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/88baa43a22a3413a95424419122a3cc9 2024-11-10T17:22:34,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/531956a277dc485d94f976bab56ff295 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/531956a277dc485d94f976bab56ff295 2024-11-10T17:22:34,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/2b7008fb89674c3dab5e982a7dcde82e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7047d3abf1e946bfbb6a68770d1e7b07, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/1b8a0f1e9d0244c2a032e13ba69525e7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8d6f88f828e942a8928f673fec51869f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8af189e2c4e14a7caecdea02e0773cb9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d557f75b037f42a6a5f20b0da6795392, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/10ab73e493b54802b5c2d0dfadec4055, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/06852bc4c0a449659e54dfefa618f4a7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fc9981e1382d4304b3821e1760df75a3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/439b26c6cef14a5bb5900739309f6bfa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/97477dd21fd94f5aadfa0e914a6780af, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9c601736d44e45168d992390e9c78d60, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3218a4e0b82241edae4e1955dd9ea81b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7b6311a3285c40a383b54e5bfa9f57b3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/03eb247987f747d0995358ba9ea1a5db, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/ba46d22105ad43f593468f00f3be66dd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/917de6c43da747078bad395f2b54c9c2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/044d10da5ee1484ba4f3105b2fac77c6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5feb181a49f04f93bf3a046cb8d47446, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/a9447c45399548f58b6161b2228a4150, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fb82b70eba1540ff9b59b9df23a9b37b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b49310a3c4ae4f228fc3636cdf849333, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/0036c84230d24fc9a362a60d25c64524, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fe133256ba054dfeb56cd9045e4b6a11, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/e27aaf631542450aa57fd549824ec220, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c10cc757183d412faac5b324977d6714, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/02af42dd510f43ac8ca32a9fef0a62dc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/bcf9983b2afd4be78d4efbb8abae33e9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3797ea29d2c54527a00c6e56fe5ad942, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9d654446f34645c6a4500eee22979a68, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/96025c7865664a53a4fb26373502b160, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7461123c41d04165b8e9d0f5242befa3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5d794bfda8914bf08d579b4266f74abc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d6a36d6e8e2e419e86af7032bbc5d4ce, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/20947b135d0f474ba14ef76ed667569b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7665ee74792f4092a9592ea06bf7c178, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c8ee725444334d56a303f0f8a940188f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/555b6a8535ea4414a165e9ca746cc877] to archive 2024-11-10T17:22:34,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:22:34,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/2b7008fb89674c3dab5e982a7dcde82e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/2b7008fb89674c3dab5e982a7dcde82e 2024-11-10T17:22:34,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7047d3abf1e946bfbb6a68770d1e7b07 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7047d3abf1e946bfbb6a68770d1e7b07 2024-11-10T17:22:34,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/1b8a0f1e9d0244c2a032e13ba69525e7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/1b8a0f1e9d0244c2a032e13ba69525e7 2024-11-10T17:22:34,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8d6f88f828e942a8928f673fec51869f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8d6f88f828e942a8928f673fec51869f 2024-11-10T17:22:34,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8af189e2c4e14a7caecdea02e0773cb9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/8af189e2c4e14a7caecdea02e0773cb9 2024-11-10T17:22:34,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d557f75b037f42a6a5f20b0da6795392 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d557f75b037f42a6a5f20b0da6795392 2024-11-10T17:22:34,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/10ab73e493b54802b5c2d0dfadec4055 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/10ab73e493b54802b5c2d0dfadec4055 2024-11-10T17:22:34,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/06852bc4c0a449659e54dfefa618f4a7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/06852bc4c0a449659e54dfefa618f4a7 2024-11-10T17:22:34,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fc9981e1382d4304b3821e1760df75a3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fc9981e1382d4304b3821e1760df75a3 2024-11-10T17:22:34,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/439b26c6cef14a5bb5900739309f6bfa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/439b26c6cef14a5bb5900739309f6bfa 2024-11-10T17:22:34,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/97477dd21fd94f5aadfa0e914a6780af to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/97477dd21fd94f5aadfa0e914a6780af 2024-11-10T17:22:34,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9c601736d44e45168d992390e9c78d60 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9c601736d44e45168d992390e9c78d60 2024-11-10T17:22:34,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3218a4e0b82241edae4e1955dd9ea81b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3218a4e0b82241edae4e1955dd9ea81b 2024-11-10T17:22:34,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7b6311a3285c40a383b54e5bfa9f57b3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7b6311a3285c40a383b54e5bfa9f57b3 2024-11-10T17:22:34,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/03eb247987f747d0995358ba9ea1a5db to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/03eb247987f747d0995358ba9ea1a5db 2024-11-10T17:22:34,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/ba46d22105ad43f593468f00f3be66dd to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/ba46d22105ad43f593468f00f3be66dd 2024-11-10T17:22:34,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/917de6c43da747078bad395f2b54c9c2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/917de6c43da747078bad395f2b54c9c2 2024-11-10T17:22:34,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/044d10da5ee1484ba4f3105b2fac77c6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/044d10da5ee1484ba4f3105b2fac77c6 2024-11-10T17:22:34,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5feb181a49f04f93bf3a046cb8d47446 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5feb181a49f04f93bf3a046cb8d47446 2024-11-10T17:22:34,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/a9447c45399548f58b6161b2228a4150 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/a9447c45399548f58b6161b2228a4150 2024-11-10T17:22:34,872 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T17:22:34,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fb82b70eba1540ff9b59b9df23a9b37b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fb82b70eba1540ff9b59b9df23a9b37b 2024-11-10T17:22:34,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b49310a3c4ae4f228fc3636cdf849333 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b49310a3c4ae4f228fc3636cdf849333 2024-11-10T17:22:34,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/0036c84230d24fc9a362a60d25c64524 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/0036c84230d24fc9a362a60d25c64524 2024-11-10T17:22:34,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fe133256ba054dfeb56cd9045e4b6a11 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/fe133256ba054dfeb56cd9045e4b6a11 2024-11-10T17:22:34,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/e27aaf631542450aa57fd549824ec220 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/e27aaf631542450aa57fd549824ec220 2024-11-10T17:22:34,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c10cc757183d412faac5b324977d6714 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c10cc757183d412faac5b324977d6714 2024-11-10T17:22:34,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/02af42dd510f43ac8ca32a9fef0a62dc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/02af42dd510f43ac8ca32a9fef0a62dc 2024-11-10T17:22:34,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/bcf9983b2afd4be78d4efbb8abae33e9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/bcf9983b2afd4be78d4efbb8abae33e9 2024-11-10T17:22:34,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3797ea29d2c54527a00c6e56fe5ad942 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/3797ea29d2c54527a00c6e56fe5ad942 2024-11-10T17:22:34,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9d654446f34645c6a4500eee22979a68 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/9d654446f34645c6a4500eee22979a68 2024-11-10T17:22:34,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/96025c7865664a53a4fb26373502b160 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/96025c7865664a53a4fb26373502b160 2024-11-10T17:22:34,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7461123c41d04165b8e9d0f5242befa3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7461123c41d04165b8e9d0f5242befa3 2024-11-10T17:22:34,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5d794bfda8914bf08d579b4266f74abc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5d794bfda8914bf08d579b4266f74abc 2024-11-10T17:22:34,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d6a36d6e8e2e419e86af7032bbc5d4ce to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/d6a36d6e8e2e419e86af7032bbc5d4ce 2024-11-10T17:22:34,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/20947b135d0f474ba14ef76ed667569b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/20947b135d0f474ba14ef76ed667569b 2024-11-10T17:22:34,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7665ee74792f4092a9592ea06bf7c178 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/7665ee74792f4092a9592ea06bf7c178 2024-11-10T17:22:34,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c8ee725444334d56a303f0f8a940188f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c8ee725444334d56a303f0f8a940188f 2024-11-10T17:22:34,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/555b6a8535ea4414a165e9ca746cc877 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/555b6a8535ea4414a165e9ca746cc877 2024-11-10T17:22:34,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/719fc7ee3ab4450a8d36365539d8fe64, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48f876ea596144f09741acc90b907c87, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/53d3bc0b5dab4df5a6dfdb5dd94443f2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48edaf96184e467aaabc5a53bfbf6c7b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/55da5ebb30b04eb486c285673a1cd318, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/027fa7cdb7814b68aab4da453546ab2e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7d18906bf04d4f65bf3c847826beddba, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/d5b2ba02e4fa4795845072f958cbd830, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/bd3b2a26c0604531bf2627f5b5a31b3f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dfd61b95230748f59a40f0889603084f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/072f32936e6e4fb4831f39ea7645fef3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a8abf8259d874930b405db23aa0f92db, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/403a3397bce14f19a40043ccd8933064, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/ffa2b105f0cb46abbfa31df2113e884e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/07f9d996a06f4dcfba470389d88289ff, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/18bfb30b66d449b6bc35a34a11ce8874, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4de913fe7f1e424bb999af23c9fa24ff, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dc7b8b2dedb343e3aaa259392b2eac7e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7a839557aaed472cbdd727a40fda893c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/99d48065f9b04b4cb977d60e157c747b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/c9069b7f304e4130a6f87f1ffed6f5dc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/54f579ed6d204440aa31a8e6d019d834, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/00ca30f77df14fc6b45913f720a5b10b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/e87a4dab370c40f7b15a9ff956216989, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/979e0e6c020542b697eb22736a5f89a2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6da3bb94089d4c5e983eadf0ef44d498, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/94e2af5f320d4a829cc2e1df6686e580, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/386a465033cc40e2a6bf8ae3564a3ef5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/9818d126d87347b3af20280e2d8aaaf8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f42b569ee35c462d827c54a832caf268, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4c17c0435b724f24a9db3d3afc52b9c9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f34c08faf4b749a6ba43b842a78c6bb4, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6f381db1911149eb8d018ff2d5b21992, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/661bb3e3b0954ce893f67f99fd69eb8e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f9c4024de03f4d9683d95deb435b138b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/b1f892c0f27c44e2a873df8f68ea0ce1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/041f7be730ed49bdad97f9776860b985, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4488ce220e7a419384a41c33792fb2cb] to archive 2024-11-10T17:22:34,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:22:34,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/719fc7ee3ab4450a8d36365539d8fe64 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/719fc7ee3ab4450a8d36365539d8fe64 2024-11-10T17:22:34,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48f876ea596144f09741acc90b907c87 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48f876ea596144f09741acc90b907c87 2024-11-10T17:22:34,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/53d3bc0b5dab4df5a6dfdb5dd94443f2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/53d3bc0b5dab4df5a6dfdb5dd94443f2 2024-11-10T17:22:34,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48edaf96184e467aaabc5a53bfbf6c7b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/48edaf96184e467aaabc5a53bfbf6c7b 2024-11-10T17:22:34,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/55da5ebb30b04eb486c285673a1cd318 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/55da5ebb30b04eb486c285673a1cd318 2024-11-10T17:22:34,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/027fa7cdb7814b68aab4da453546ab2e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/027fa7cdb7814b68aab4da453546ab2e 2024-11-10T17:22:34,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7d18906bf04d4f65bf3c847826beddba to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7d18906bf04d4f65bf3c847826beddba 2024-11-10T17:22:34,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/d5b2ba02e4fa4795845072f958cbd830 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/d5b2ba02e4fa4795845072f958cbd830 2024-11-10T17:22:34,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/bd3b2a26c0604531bf2627f5b5a31b3f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/bd3b2a26c0604531bf2627f5b5a31b3f 2024-11-10T17:22:34,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dfd61b95230748f59a40f0889603084f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dfd61b95230748f59a40f0889603084f 2024-11-10T17:22:34,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/072f32936e6e4fb4831f39ea7645fef3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/072f32936e6e4fb4831f39ea7645fef3 2024-11-10T17:22:34,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a8abf8259d874930b405db23aa0f92db to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a8abf8259d874930b405db23aa0f92db 2024-11-10T17:22:34,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/403a3397bce14f19a40043ccd8933064 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/403a3397bce14f19a40043ccd8933064 2024-11-10T17:22:34,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/ffa2b105f0cb46abbfa31df2113e884e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/ffa2b105f0cb46abbfa31df2113e884e 2024-11-10T17:22:34,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/07f9d996a06f4dcfba470389d88289ff to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/07f9d996a06f4dcfba470389d88289ff 2024-11-10T17:22:34,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/18bfb30b66d449b6bc35a34a11ce8874 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/18bfb30b66d449b6bc35a34a11ce8874 2024-11-10T17:22:34,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4de913fe7f1e424bb999af23c9fa24ff to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4de913fe7f1e424bb999af23c9fa24ff 2024-11-10T17:22:34,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dc7b8b2dedb343e3aaa259392b2eac7e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/dc7b8b2dedb343e3aaa259392b2eac7e 2024-11-10T17:22:34,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7a839557aaed472cbdd727a40fda893c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/7a839557aaed472cbdd727a40fda893c 2024-11-10T17:22:34,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/99d48065f9b04b4cb977d60e157c747b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/99d48065f9b04b4cb977d60e157c747b 2024-11-10T17:22:34,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/c9069b7f304e4130a6f87f1ffed6f5dc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/c9069b7f304e4130a6f87f1ffed6f5dc 2024-11-10T17:22:34,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/54f579ed6d204440aa31a8e6d019d834 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/54f579ed6d204440aa31a8e6d019d834 2024-11-10T17:22:34,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/00ca30f77df14fc6b45913f720a5b10b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/00ca30f77df14fc6b45913f720a5b10b 2024-11-10T17:22:34,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/e87a4dab370c40f7b15a9ff956216989 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/e87a4dab370c40f7b15a9ff956216989 2024-11-10T17:22:34,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/979e0e6c020542b697eb22736a5f89a2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/979e0e6c020542b697eb22736a5f89a2 2024-11-10T17:22:34,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6da3bb94089d4c5e983eadf0ef44d498 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6da3bb94089d4c5e983eadf0ef44d498 2024-11-10T17:22:34,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/94e2af5f320d4a829cc2e1df6686e580 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/94e2af5f320d4a829cc2e1df6686e580 2024-11-10T17:22:34,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/386a465033cc40e2a6bf8ae3564a3ef5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/386a465033cc40e2a6bf8ae3564a3ef5 2024-11-10T17:22:34,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/9818d126d87347b3af20280e2d8aaaf8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/9818d126d87347b3af20280e2d8aaaf8 2024-11-10T17:22:34,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f42b569ee35c462d827c54a832caf268 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f42b569ee35c462d827c54a832caf268 2024-11-10T17:22:34,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4c17c0435b724f24a9db3d3afc52b9c9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4c17c0435b724f24a9db3d3afc52b9c9 2024-11-10T17:22:34,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f34c08faf4b749a6ba43b842a78c6bb4 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f34c08faf4b749a6ba43b842a78c6bb4 2024-11-10T17:22:34,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6f381db1911149eb8d018ff2d5b21992 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/6f381db1911149eb8d018ff2d5b21992 2024-11-10T17:22:34,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/661bb3e3b0954ce893f67f99fd69eb8e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/661bb3e3b0954ce893f67f99fd69eb8e 2024-11-10T17:22:34,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f9c4024de03f4d9683d95deb435b138b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/f9c4024de03f4d9683d95deb435b138b 2024-11-10T17:22:34,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/b1f892c0f27c44e2a873df8f68ea0ce1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/b1f892c0f27c44e2a873df8f68ea0ce1 2024-11-10T17:22:34,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/041f7be730ed49bdad97f9776860b985 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/041f7be730ed49bdad97f9776860b985 2024-11-10T17:22:34,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4488ce220e7a419384a41c33792fb2cb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/4488ce220e7a419384a41c33792fb2cb 2024-11-10T17:22:34,957 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/recovered.edits/594.seqid, newMaxSeqId=594, maxSeqId=1 2024-11-10T17:22:34,958 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. 2024-11-10T17:22:34,958 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 86036b2e5b1a6747cefd43fa6b653d49: Waiting for close lock at 1731259353479Running coprocessor pre-close hooks at 1731259353479Disabling compacts and flushes for region at 1731259353479Disabling writes for close at 1731259353518 (+39 ms)Obtaining lock to block concurrent updates at 1731259353519 (+1 ms)Preparing flush snapshotting stores in 86036b2e5b1a6747cefd43fa6b653d49 at 1731259353519Finished memstore snapshotting TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731259353519Flushing stores of TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49. at 1731259353519Flushing 86036b2e5b1a6747cefd43fa6b653d49/A: creating writer at 1731259353519Flushing 86036b2e5b1a6747cefd43fa6b653d49/A: appending metadata at 1731259353522 (+3 ms)Flushing 86036b2e5b1a6747cefd43fa6b653d49/A: closing flushed file at 1731259353522Flushing 86036b2e5b1a6747cefd43fa6b653d49/B: creating writer at 1731259353932 (+410 ms)Flushing 86036b2e5b1a6747cefd43fa6b653d49/B: appending metadata at 1731259353935 (+3 ms)Flushing 86036b2e5b1a6747cefd43fa6b653d49/B: closing flushed file at 1731259353935Flushing 86036b2e5b1a6747cefd43fa6b653d49/C: creating writer at 1731259354345 (+410 ms)Flushing 86036b2e5b1a6747cefd43fa6b653d49/C: appending metadata at 1731259354347 (+2 ms)Flushing 86036b2e5b1a6747cefd43fa6b653d49/C: closing flushed file at 1731259354348 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19786158: reopening flushed file at 1731259354757 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15f49da1: reopening flushed file at 1731259354762 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f3f6847: reopening flushed file at 1731259354767 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 86036b2e5b1a6747cefd43fa6b653d49 in 1254ms, sequenceid=591, compaction requested=true at 1731259354773 (+6 ms)Writing region close event to WAL at 1731259354950 (+177 ms)Running coprocessor post-close hooks at 1731259354958 (+8 ms)Closed at 1731259354958 2024-11-10T17:22:34,962 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:34,963 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=86036b2e5b1a6747cefd43fa6b653d49, regionState=CLOSED 2024-11-10T17:22:34,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:22:34,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T17:22:34,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 86036b2e5b1a6747cefd43fa6b653d49, server=8e3de9160f9f,36421,1731259326553 in 1.6450 sec 2024-11-10T17:22:34,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-10T17:22:34,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86036b2e5b1a6747cefd43fa6b653d49, UNASSIGN in 1.6530 sec 2024-11-10T17:22:34,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T17:22:34,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6610 sec 2024-11-10T17:22:34,975 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259354974"}]},"ts":"1731259354974"} 2024-11-10T17:22:34,976 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T17:22:34,976 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T17:22:34,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6770 sec 2024-11-10T17:22:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T17:22:35,452 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:35,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T17:22:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:35,461 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T17:22:35,462 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:35,469 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:35,473 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/recovered.edits] 2024-11-10T17:22:35,477 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/086545f2f0ee468aa1c145a669d42097 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/086545f2f0ee468aa1c145a669d42097 2024-11-10T17:22:35,478 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/621add3028684f5bba35c77bafc49ed5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/621add3028684f5bba35c77bafc49ed5 2024-11-10T17:22:35,480 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8c62dc47290e4203b1bf461f70ffb15c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/8c62dc47290e4203b1bf461f70ffb15c 2024-11-10T17:22:35,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/eeb920e502b740ad833734b3bb5ba933 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/A/eeb920e502b740ad833734b3bb5ba933 2024-11-10T17:22:35,484 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5286040e3f53483daa16d56650a6df0a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/5286040e3f53483daa16d56650a6df0a 2024-11-10T17:22:35,485 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/6eaa1debcea348f8a6c0d24d348fa695 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/6eaa1debcea348f8a6c0d24d348fa695 2024-11-10T17:22:35,487 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b415870d30324c1f9dcd91927dbb725f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/b415870d30324c1f9dcd91927dbb725f 2024-11-10T17:22:35,488 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c3c9acc9573c47c6b9ec6f0a710b27ac to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/B/c3c9acc9573c47c6b9ec6f0a710b27ac 2024-11-10T17:22:35,491 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/47bb9be5e1ae48f0922a147a69e5b2a1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/47bb9be5e1ae48f0922a147a69e5b2a1 2024-11-10T17:22:35,492 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/82f9c7e1eb8e49649baae98c819b86fb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/82f9c7e1eb8e49649baae98c819b86fb 2024-11-10T17:22:35,494 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/90d7f2083a4742d8b8819c6f16bf1739 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/90d7f2083a4742d8b8819c6f16bf1739 2024-11-10T17:22:35,495 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a989667ec51a44da9251baf5095aadda to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/C/a989667ec51a44da9251baf5095aadda 2024-11-10T17:22:35,499 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/recovered.edits/594.seqid to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49/recovered.edits/594.seqid 2024-11-10T17:22:35,499 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/86036b2e5b1a6747cefd43fa6b653d49 2024-11-10T17:22:35,500 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T17:22:35,506 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:35,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T17:22:35,521 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T17:22:35,558 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T17:22:35,560 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:35,560 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T17:22:35,560 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731259355560"}]},"ts":"9223372036854775807"} 2024-11-10T17:22:35,564 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T17:22:35,564 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 86036b2e5b1a6747cefd43fa6b653d49, NAME => 'TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T17:22:35,565 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T17:22:35,565 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731259355565"}]},"ts":"9223372036854775807"} 2024-11-10T17:22:35,568 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T17:22:35,569 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:35,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-11-10T17:22:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T17:22:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T17:22:35,572 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:35,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-255130881_22 at /127.0.0.1:39678 [Waiting for operation #338] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;8e3de9160f9f:36421-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2144637464_22 at /127.0.0.1:42410 [Waiting for operation #124] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-255130881_22 at /127.0.0.1:36664 [Waiting for operation #328] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 258) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6336 (was 6903) 2024-11-10T17:22:35,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=6335 2024-11-10T17:22:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:22:35,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:22:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:35,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T17:22:35,605 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:35,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-10T17:22:35,606 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T17:22:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-10T17:22:35,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741963_1139 (size=959) 2024-11-10T17:22:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-10T17:22:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-10T17:22:36,015 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:22:36,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741964_1140 (size=53) 2024-11-10T17:22:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-10T17:22:36,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T17:22:36,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:36,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing afd93882106705ea245d285e044c07aa, disabling compactions & flushes 2024-11-10T17:22:36,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:36,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:36,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. after waiting 0 ms 2024-11-10T17:22:36,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:36,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:36,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for afd93882106705ea245d285e044c07aa: Waiting for close lock at 1731259356422Disabling compacts and flushes for region at 1731259356422Disabling writes for close at 1731259356422Writing region close event to WAL at 1731259356422Closed at 1731259356422 2024-11-10T17:22:36,423 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T17:22:36,424 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731259356423"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731259356423"}]},"ts":"1731259356423"} 2024-11-10T17:22:36,426 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T17:22:36,427 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T17:22:36,427 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259356427"}]},"ts":"1731259356427"} 2024-11-10T17:22:36,428 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T17:22:36,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, ASSIGN}] 2024-11-10T17:22:36,430 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, ASSIGN 2024-11-10T17:22:36,430 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, ASSIGN; state=OFFLINE, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=false 2024-11-10T17:22:36,581 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:36,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, ASSIGN because future has completed 2024-11-10T17:22:36,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:22:36,740 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:36,740 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:22:36,740 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,740 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:36,741 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,741 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-10T17:22:36,742 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,745 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd93882106705ea245d285e044c07aa columnFamilyName A 2024-11-10T17:22:36,745 DEBUG [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:36,746 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(327): Store=afd93882106705ea245d285e044c07aa/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:36,746 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,747 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd93882106705ea245d285e044c07aa columnFamilyName B 2024-11-10T17:22:36,747 DEBUG [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:36,748 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(327): Store=afd93882106705ea245d285e044c07aa/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:36,748 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,749 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd93882106705ea245d285e044c07aa columnFamilyName C 2024-11-10T17:22:36,749 DEBUG [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:36,750 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(327): Store=afd93882106705ea245d285e044c07aa/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:36,750 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,751 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,751 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,752 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,752 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,753 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:22:36,754 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,755 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:22:36,756 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened afd93882106705ea245d285e044c07aa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61460399, jitterRate=-0.0841686874628067}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:22:36,756 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:36,756 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for afd93882106705ea245d285e044c07aa: Running coprocessor pre-open hook at 1731259356741Writing region info on filesystem at 1731259356741Initializing all the Stores at 1731259356742 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259356742Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259356742Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259356742Cleaning up temporary data from old regions at 1731259356752 (+10 ms)Running coprocessor post-open hooks at 1731259356756 (+4 ms)Region opened successfully at 1731259356756 2024-11-10T17:22:36,757 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., pid=28, masterSystemTime=1731259356736 2024-11-10T17:22:36,759 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:36,759 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:36,760 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:36,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:22:36,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T17:22:36,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 in 179 msec 2024-11-10T17:22:36,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-10T17:22:36,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, ASSIGN in 336 msec 2024-11-10T17:22:36,767 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T17:22:36,767 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259356767"}]},"ts":"1731259356767"} 2024-11-10T17:22:36,769 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T17:22:36,770 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T17:22:36,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-10T17:22:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-10T17:22:37,752 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:37,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:37,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:37,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:37,754 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edbf8bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:37,757 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:37,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6250a25e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:37,758 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:37,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:37,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T17:22:37,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,40007,1731259325831 2024-11-10T17:22:37,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@764d02d6 2024-11-10T17:22:37,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T17:22:37,764 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T17:22:37,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:22:37,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:22:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T17:22:37,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741965_1141 (size=995) 2024-11-10T17:22:38,189 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T17:22:38,189 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T17:22:38,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:22:38,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, REOPEN/MOVE}] 2024-11-10T17:22:38,200 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, REOPEN/MOVE 2024-11-10T17:22:38,201 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, REOPEN/MOVE because future has completed 2024-11-10T17:22:38,203 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:22:38,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:22:38,356 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,357 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:22:38,357 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing afd93882106705ea245d285e044c07aa, disabling compactions & flushes 2024-11-10T17:22:38,357 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,357 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,357 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. after waiting 0 ms 2024-11-10T17:22:38,357 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,362 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T17:22:38,362 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,362 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for afd93882106705ea245d285e044c07aa: Waiting for close lock at 1731259358357Running coprocessor pre-close hooks at 1731259358357Disabling compacts and flushes for region at 1731259358357Disabling writes for close at 1731259358357Writing region close event to WAL at 1731259358358 (+1 ms)Running coprocessor post-close hooks at 1731259358362 (+4 ms)Closed at 1731259358362 2024-11-10T17:22:38,362 WARN [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: afd93882106705ea245d285e044c07aa to self. 2024-11-10T17:22:38,365 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,365 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=CLOSED 2024-11-10T17:22:38,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:22:38,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T17:22:38,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 in 164 msec 2024-11-10T17:22:38,370 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, REOPEN/MOVE; state=CLOSED, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=true 2024-11-10T17:22:38,520 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, REOPEN/MOVE because future has completed 2024-11-10T17:22:38,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:22:38,678 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,679 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:22:38,679 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,679 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:22:38,679 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,679 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,684 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,689 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd93882106705ea245d285e044c07aa columnFamilyName A 2024-11-10T17:22:38,691 DEBUG [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:38,691 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(327): Store=afd93882106705ea245d285e044c07aa/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:38,692 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,692 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd93882106705ea245d285e044c07aa columnFamilyName B 2024-11-10T17:22:38,692 DEBUG [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:38,693 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(327): Store=afd93882106705ea245d285e044c07aa/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:38,693 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,693 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd93882106705ea245d285e044c07aa columnFamilyName C 2024-11-10T17:22:38,693 DEBUG [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:38,694 INFO [StoreOpener-afd93882106705ea245d285e044c07aa-1 {}] regionserver.HStore(327): Store=afd93882106705ea245d285e044c07aa/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:22:38,694 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,695 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,696 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,697 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,697 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,697 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:22:38,698 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,699 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened afd93882106705ea245d285e044c07aa; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64238714, jitterRate=-0.04276856780052185}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:22:38,699 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,700 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for afd93882106705ea245d285e044c07aa: Running coprocessor pre-open hook at 1731259358679Writing region info on filesystem at 1731259358679Initializing all the Stores at 1731259358680 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259358680Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259358680Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259358680Cleaning up temporary data from old regions at 1731259358697 (+17 ms)Running coprocessor post-open hooks at 1731259358699 (+2 ms)Region opened successfully at 1731259358700 (+1 ms) 2024-11-10T17:22:38,700 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., pid=33, masterSystemTime=1731259358675 2024-11-10T17:22:38,702 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,702 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,703 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=OPEN, openSeqNum=5, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:22:38,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-10T17:22:38,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 in 183 msec 2024-11-10T17:22:38,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-10T17:22:38,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, REOPEN/MOVE in 508 msec 2024-11-10T17:22:38,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-10T17:22:38,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-10T17:22:38,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-11-10T17:22:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T17:22:38,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:38,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,719 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a74d176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,721 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b920ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,723 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61627e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,725 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e66dac8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,727 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d162cb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b55bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,731 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a2e8a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9db4ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,734 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765fbc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,736 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca92c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,738 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af7f9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,739 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64586342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@278dcaf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,744 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7077c1b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,748 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21264392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,749 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f7e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:22:38,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:22:38,753 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:22:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:22:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:22:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@452afc7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:22:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:22:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:38,755 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:22:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556223ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:22:38,757 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,757 DEBUG [Thread-701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,757 DEBUG [Thread-703 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,765 DEBUG [Thread-706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:38,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:38,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:38,777 DEBUG [Thread-710 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,778 DEBUG [Thread-710 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-10T17:22:38,778 DEBUG [Thread-710 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T17:22:38,781 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:38,782 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:38,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,789 DEBUG [Thread-708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,821 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,829 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110073961ef7c134087ae6607cd8249623b_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259358773/Put/seqid=0 2024-11-10T17:22:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259418843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259418844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259418846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,852 DEBUG [Thread-712 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:22:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:22:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:22:38,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:22:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:22:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259418849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259418852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741966_1142 (size=12154) 2024-11-10T17:22:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T17:22:38,892 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:38,898 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110073961ef7c134087ae6607cd8249623b_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110073961ef7c134087ae6607cd8249623b_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:38,899 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/faef8c01864748449f242a9cacab4bcd, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:38,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/faef8c01864748449f242a9cacab4bcd is 175, key is test_row_0/A:col10/1731259358773/Put/seqid=0 2024-11-10T17:22:38,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741967_1143 (size=30955) 2024-11-10T17:22:38,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T17:22:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:38,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:38,948 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/faef8c01864748449f242a9cacab4bcd 2024-11-10T17:22:38,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259418962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259418962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259418962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259418962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259418962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/cbf529ddb4d942eea0cd43a68b75cfe0 is 50, key is test_row_0/B:col10/1731259358773/Put/seqid=0 2024-11-10T17:22:39,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741968_1144 (size=12001) 2024-11-10T17:22:39,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T17:22:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:39,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T17:22:39,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259419172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259419172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259419173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259419172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259419173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T17:22:39,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:39,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T17:22:39,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:39,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:39,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T17:22:39,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:39,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/cbf529ddb4d942eea0cd43a68b75cfe0 2024-11-10T17:22:39,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/b6702d4a3c484dcba75a314998cad4aa is 50, key is test_row_0/C:col10/1731259358773/Put/seqid=0 2024-11-10T17:22:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741969_1145 (size=12001) 2024-11-10T17:22:39,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/b6702d4a3c484dcba75a314998cad4aa 2024-11-10T17:22:39,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259419482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259419482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259419482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259419482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259419482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:39,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/faef8c01864748449f242a9cacab4bcd as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/faef8c01864748449f242a9cacab4bcd 2024-11-10T17:22:39,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/faef8c01864748449f242a9cacab4bcd, entries=150, sequenceid=17, filesize=30.2 K 2024-11-10T17:22:39,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/cbf529ddb4d942eea0cd43a68b75cfe0 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/cbf529ddb4d942eea0cd43a68b75cfe0 2024-11-10T17:22:39,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/cbf529ddb4d942eea0cd43a68b75cfe0, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T17:22:39,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/b6702d4a3c484dcba75a314998cad4aa as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b6702d4a3c484dcba75a314998cad4aa 2024-11-10T17:22:39,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b6702d4a3c484dcba75a314998cad4aa, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T17:22:39,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for afd93882106705ea245d285e044c07aa in 749ms, sequenceid=17, compaction requested=false 2024-11-10T17:22:39,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:39,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T17:22:39,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:39,568 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T17:22:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111061f7c39bf7004a2eb3095449ff03b652_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259358840/Put/seqid=0 2024-11-10T17:22:39,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741970_1146 (size=12154) 2024-11-10T17:22:39,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:39,647 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111061f7c39bf7004a2eb3095449ff03b652_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061f7c39bf7004a2eb3095449ff03b652_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/f4857245ef344549b611b27b92890e89, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/f4857245ef344549b611b27b92890e89 is 175, key is test_row_0/A:col10/1731259358840/Put/seqid=0 2024-11-10T17:22:39,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741971_1147 (size=30955) 2024-11-10T17:22:39,689 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/f4857245ef344549b611b27b92890e89 2024-11-10T17:22:39,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/3a0fa5e1dff245b1b0f484270fb672cb is 50, key is test_row_0/B:col10/1731259358840/Put/seqid=0 2024-11-10T17:22:39,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741972_1148 (size=12001) 2024-11-10T17:22:39,736 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/3a0fa5e1dff245b1b0f484270fb672cb 2024-11-10T17:22:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/46dfff49765542a394aff8b8649efa76 is 50, key is test_row_0/C:col10/1731259358840/Put/seqid=0 2024-11-10T17:22:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741973_1149 (size=12001) 2024-11-10T17:22:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T17:22:40,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:40,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259420017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259420020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259420022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259420024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259420037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259420132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259420142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259420142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259420143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259420152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,194 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/46dfff49765542a394aff8b8649efa76 2024-11-10T17:22:40,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/f4857245ef344549b611b27b92890e89 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/f4857245ef344549b611b27b92890e89 2024-11-10T17:22:40,209 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/f4857245ef344549b611b27b92890e89, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T17:22:40,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/3a0fa5e1dff245b1b0f484270fb672cb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/3a0fa5e1dff245b1b0f484270fb672cb 2024-11-10T17:22:40,217 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/3a0fa5e1dff245b1b0f484270fb672cb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T17:22:40,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/46dfff49765542a394aff8b8649efa76 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46dfff49765542a394aff8b8649efa76 2024-11-10T17:22:40,230 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46dfff49765542a394aff8b8649efa76, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T17:22:40,233 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for afd93882106705ea245d285e044c07aa in 665ms, sequenceid=40, compaction requested=false 2024-11-10T17:22:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:40,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-10T17:22:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-10T17:22:40,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-10T17:22:40,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4540 sec 2024-11-10T17:22:40,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.4630 sec 2024-11-10T17:22:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:40,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T17:22:40,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100f6cf6b36a904dd4aab187b34ffcf830_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259360024/Put/seqid=0 2024-11-10T17:22:40,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741974_1150 (size=12154) 2024-11-10T17:22:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259420393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259420395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259420396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259420397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259420397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259420501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259420511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259420511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259420512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259420512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,527 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T17:22:40,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259420711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259420722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259420722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259420722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259420722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:40,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:40,793 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100f6cf6b36a904dd4aab187b34ffcf830_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100f6cf6b36a904dd4aab187b34ffcf830_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:40,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aaa987f67d464959a459d31b0b8e4b58, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:40,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aaa987f67d464959a459d31b0b8e4b58 is 175, key is test_row_0/A:col10/1731259360024/Put/seqid=0 2024-11-10T17:22:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741975_1151 (size=30955) 2024-11-10T17:22:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T17:22:40,933 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:40,933 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-10T17:22:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T17:22:40,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:40,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:40,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:41,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259421021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259421032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259421032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259421032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259421032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T17:22:41,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T17:22:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:41,229 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aaa987f67d464959a459d31b0b8e4b58 2024-11-10T17:22:41,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ceafe0e310054ec19c784dfd859f5ed1 is 50, key is test_row_0/B:col10/1731259360024/Put/seqid=0 2024-11-10T17:22:41,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T17:22:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T17:22:41,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741976_1152 (size=12001) 2024-11-10T17:22:41,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ceafe0e310054ec19c784dfd859f5ed1 2024-11-10T17:22:41,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/fbbd3a493e0942daaba7ba7f15ccb9cd is 50, key is test_row_0/C:col10/1731259360024/Put/seqid=0 2024-11-10T17:22:41,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741977_1153 (size=12001) 2024-11-10T17:22:41,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/fbbd3a493e0942daaba7ba7f15ccb9cd 2024-11-10T17:22:41,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aaa987f67d464959a459d31b0b8e4b58 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aaa987f67d464959a459d31b0b8e4b58 2024-11-10T17:22:41,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aaa987f67d464959a459d31b0b8e4b58, entries=150, sequenceid=54, filesize=30.2 K 2024-11-10T17:22:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ceafe0e310054ec19c784dfd859f5ed1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ceafe0e310054ec19c784dfd859f5ed1 2024-11-10T17:22:41,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ceafe0e310054ec19c784dfd859f5ed1, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T17:22:41,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/fbbd3a493e0942daaba7ba7f15ccb9cd as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/fbbd3a493e0942daaba7ba7f15ccb9cd 2024-11-10T17:22:41,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/fbbd3a493e0942daaba7ba7f15ccb9cd, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T17:22:41,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for afd93882106705ea245d285e044c07aa in 991ms, sequenceid=54, compaction requested=true 2024-11-10T17:22:41,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:41,337 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:41,338 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:41,344 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:41,344 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:41,345 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:41,345 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:41,345 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,345 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,345 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/cbf529ddb4d942eea0cd43a68b75cfe0, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/3a0fa5e1dff245b1b0f484270fb672cb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ceafe0e310054ec19c784dfd859f5ed1] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=35.2 K 2024-11-10T17:22:41,345 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/faef8c01864748449f242a9cacab4bcd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/f4857245ef344549b611b27b92890e89, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aaa987f67d464959a459d31b0b8e4b58] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=90.7 K 2024-11-10T17:22:41,345 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,345 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/faef8c01864748449f242a9cacab4bcd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/f4857245ef344549b611b27b92890e89, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aaa987f67d464959a459d31b0b8e4b58] 2024-11-10T17:22:41,346 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting faef8c01864748449f242a9cacab4bcd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731259358773 2024-11-10T17:22:41,347 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4857245ef344549b611b27b92890e89, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259358840 2024-11-10T17:22:41,348 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting cbf529ddb4d942eea0cd43a68b75cfe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731259358773 2024-11-10T17:22:41,349 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a0fa5e1dff245b1b0f484270fb672cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259358840 2024-11-10T17:22:41,349 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaa987f67d464959a459d31b0b8e4b58, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731259360015 2024-11-10T17:22:41,350 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting ceafe0e310054ec19c784dfd859f5ed1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731259360015 2024-11-10T17:22:41,360 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,366 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:41,367 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/c690d882394e44cfa72ea5c1775ef439 is 50, key is test_row_0/B:col10/1731259360024/Put/seqid=0 2024-11-10T17:22:41,368 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ab5b9bf206e045d5a3040024d482a27b_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,376 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ab5b9bf206e045d5a3040024d482a27b_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,376 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ab5b9bf206e045d5a3040024d482a27b_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T17:22:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741979_1155 (size=4469) 2024-11-10T17:22:41,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T17:22:41,403 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#135 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:41,404 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4d396149be8c4b2cac497b1b4e118ce3 is 175, key is test_row_0/A:col10/1731259360024/Put/seqid=0 2024-11-10T17:22:41,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741978_1154 (size=12104) 2024-11-10T17:22:41,419 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/c690d882394e44cfa72ea5c1775ef439 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c690d882394e44cfa72ea5c1775ef439 2024-11-10T17:22:41,427 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into c690d882394e44cfa72ea5c1775ef439(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:41,427 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:41,427 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=13, startTime=1731259361338; duration=0sec 2024-11-10T17:22:41,428 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:41,428 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:41,428 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ec989e5550de4c94bfa3eca693b8ad87_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259360395/Put/seqid=0 2024-11-10T17:22:41,430 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:41,431 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:41,431 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,431 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b6702d4a3c484dcba75a314998cad4aa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46dfff49765542a394aff8b8649efa76, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/fbbd3a493e0942daaba7ba7f15ccb9cd] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=35.2 K 2024-11-10T17:22:41,431 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b6702d4a3c484dcba75a314998cad4aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731259358773 2024-11-10T17:22:41,432 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 46dfff49765542a394aff8b8649efa76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259358840 2024-11-10T17:22:41,432 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting fbbd3a493e0942daaba7ba7f15ccb9cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731259360015 2024-11-10T17:22:41,448 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:41,449 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/cd04162849f34fa1a72e910f8cd6c4ed is 50, key is test_row_0/C:col10/1731259360024/Put/seqid=0 2024-11-10T17:22:41,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741980_1156 (size=31058) 2024-11-10T17:22:41,462 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4d396149be8c4b2cac497b1b4e118ce3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d396149be8c4b2cac497b1b4e118ce3 2024-11-10T17:22:41,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741981_1157 (size=12154) 2024-11-10T17:22:41,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:41,471 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into 4d396149be8c4b2cac497b1b4e118ce3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:41,471 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:41,471 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=13, startTime=1731259361337; duration=0sec 2024-11-10T17:22:41,471 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:41,471 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:41,476 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ec989e5550de4c94bfa3eca693b8ad87_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ec989e5550de4c94bfa3eca693b8ad87_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:41,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741982_1158 (size=12104) 2024-11-10T17:22:41,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/b3484b8a054f4e2292b559b85a5864b6, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/b3484b8a054f4e2292b559b85a5864b6 is 175, key is test_row_0/A:col10/1731259360395/Put/seqid=0 2024-11-10T17:22:41,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741983_1159 (size=30955) 2024-11-10T17:22:41,516 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/b3484b8a054f4e2292b559b85a5864b6 2024-11-10T17:22:41,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ff12dd22f5344f649c76ec5dc8e6a652 is 50, key is test_row_0/B:col10/1731259360395/Put/seqid=0 2024-11-10T17:22:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741984_1160 (size=12001) 2024-11-10T17:22:41,553 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ff12dd22f5344f649c76ec5dc8e6a652 2024-11-10T17:22:41,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259421555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259421557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T17:22:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259421559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259421561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259421562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/46be85c77e984dc68d8e04be7136fd0e is 50, key is test_row_0/C:col10/1731259360395/Put/seqid=0 2024-11-10T17:22:41,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741985_1161 (size=12001) 2024-11-10T17:22:41,593 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/46be85c77e984dc68d8e04be7136fd0e 2024-11-10T17:22:41,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/b3484b8a054f4e2292b559b85a5864b6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/b3484b8a054f4e2292b559b85a5864b6 2024-11-10T17:22:41,608 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/b3484b8a054f4e2292b559b85a5864b6, entries=150, sequenceid=77, filesize=30.2 K 2024-11-10T17:22:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ff12dd22f5344f649c76ec5dc8e6a652 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ff12dd22f5344f649c76ec5dc8e6a652 2024-11-10T17:22:41,637 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ff12dd22f5344f649c76ec5dc8e6a652, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T17:22:41,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/46be85c77e984dc68d8e04be7136fd0e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46be85c77e984dc68d8e04be7136fd0e 2024-11-10T17:22:41,656 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46be85c77e984dc68d8e04be7136fd0e, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T17:22:41,657 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for afd93882106705ea245d285e044c07aa in 256ms, sequenceid=77, compaction requested=false 2024-11-10T17:22:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-10T17:22:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-10T17:22:41,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-10T17:22:41,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 720 msec 2024-11-10T17:22:41,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 733 msec 2024-11-10T17:22:41,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:41,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T17:22:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:41,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105c801e6c6d3345fdaef4e4673670b22d_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259361557/Put/seqid=0 2024-11-10T17:22:41,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259421718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259421720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259421726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741986_1162 (size=14594) 2024-11-10T17:22:41,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:41,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259421727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,741 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105c801e6c6d3345fdaef4e4673670b22d_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105c801e6c6d3345fdaef4e4673670b22d_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259421729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/fd2f1bcc2ee74252a06d2a9ebf196d13, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/fd2f1bcc2ee74252a06d2a9ebf196d13 is 175, key is test_row_0/A:col10/1731259361557/Put/seqid=0 2024-11-10T17:22:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741987_1163 (size=39549) 2024-11-10T17:22:41,756 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/fd2f1bcc2ee74252a06d2a9ebf196d13 2024-11-10T17:22:41,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d5235d9d08744a72b15d825816f0cfd8 is 50, key is test_row_0/B:col10/1731259361557/Put/seqid=0 2024-11-10T17:22:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741988_1164 (size=12001) 2024-11-10T17:22:41,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d5235d9d08744a72b15d825816f0cfd8 2024-11-10T17:22:41,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/4d07f414a3554dc4957ee41e908ebd2b is 50, key is test_row_0/C:col10/1731259361557/Put/seqid=0 2024-11-10T17:22:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741989_1165 (size=12001) 2024-11-10T17:22:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259421831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259421832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259421832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/4d07f414a3554dc4957ee41e908ebd2b 2024-11-10T17:22:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259421842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/fd2f1bcc2ee74252a06d2a9ebf196d13 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/fd2f1bcc2ee74252a06d2a9ebf196d13 2024-11-10T17:22:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/fd2f1bcc2ee74252a06d2a9ebf196d13, entries=200, sequenceid=96, filesize=38.6 K 2024-11-10T17:22:41,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d5235d9d08744a72b15d825816f0cfd8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d5235d9d08744a72b15d825816f0cfd8 2024-11-10T17:22:41,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d5235d9d08744a72b15d825816f0cfd8, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T17:22:41,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259421852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:41,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/4d07f414a3554dc4957ee41e908ebd2b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4d07f414a3554dc4957ee41e908ebd2b 2024-11-10T17:22:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:41,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4d07f414a3554dc4957ee41e908ebd2b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T17:22:41,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=120.76 KB/123660 for afd93882106705ea245d285e044c07aa in 192ms, sequenceid=96, compaction requested=true 2024-11-10T17:22:41,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:41,868 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:41,869 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:41,870 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:41,870 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,870 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d396149be8c4b2cac497b1b4e118ce3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/b3484b8a054f4e2292b559b85a5864b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/fd2f1bcc2ee74252a06d2a9ebf196d13] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=99.2 K 2024-11-10T17:22:41,870 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,870 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d396149be8c4b2cac497b1b4e118ce3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/b3484b8a054f4e2292b559b85a5864b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/fd2f1bcc2ee74252a06d2a9ebf196d13] 2024-11-10T17:22:41,870 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d396149be8c4b2cac497b1b4e118ce3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731259360015 2024-11-10T17:22:41,871 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3484b8a054f4e2292b559b85a5864b6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731259360393 2024-11-10T17:22:41,871 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd2f1bcc2ee74252a06d2a9ebf196d13, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731259361557 2024-11-10T17:22:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T17:22:41,879 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,885 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102b3385f05fe442bb994f84db338b3dee_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,887 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/cd04162849f34fa1a72e910f8cd6c4ed as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/cd04162849f34fa1a72e910f8cd6c4ed 2024-11-10T17:22:41,889 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102b3385f05fe442bb994f84db338b3dee_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,890 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102b3385f05fe442bb994f84db338b3dee_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:41,898 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into cd04162849f34fa1a72e910f8cd6c4ed(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:41,898 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:41,898 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=13, startTime=1731259361339; duration=0sec 2024-11-10T17:22:41,898 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T17:22:41,898 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:41,898 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:41,898 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:41,900 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:41,900 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:41,900 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:41,900 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c690d882394e44cfa72ea5c1775ef439, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ff12dd22f5344f649c76ec5dc8e6a652, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d5235d9d08744a72b15d825816f0cfd8] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=35.3 K 2024-11-10T17:22:41,901 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c690d882394e44cfa72ea5c1775ef439, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731259360015 2024-11-10T17:22:41,901 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting ff12dd22f5344f649c76ec5dc8e6a652, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731259360393 2024-11-10T17:22:41,902 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d5235d9d08744a72b15d825816f0cfd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731259361557 2024-11-10T17:22:41,928 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#145 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:41,928 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d291bf7ec7a747bdb373b9f0dcb81882 is 50, key is test_row_0/B:col10/1731259361557/Put/seqid=0 2024-11-10T17:22:41,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741990_1166 (size=4469) 2024-11-10T17:22:41,948 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#144 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:41,949 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/27dab2c7796846d1b9ff09b6251cc992 is 175, key is test_row_0/A:col10/1731259361557/Put/seqid=0 2024-11-10T17:22:41,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741991_1167 (size=12207) 2024-11-10T17:22:41,987 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d291bf7ec7a747bdb373b9f0dcb81882 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d291bf7ec7a747bdb373b9f0dcb81882 2024-11-10T17:22:42,001 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into d291bf7ec7a747bdb373b9f0dcb81882(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:42,001 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:42,001 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=13, startTime=1731259361873; duration=0sec 2024-11-10T17:22:42,001 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:42,001 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:42,001 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:42,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741992_1168 (size=31161) 2024-11-10T17:22:42,009 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:42,014 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/27dab2c7796846d1b9ff09b6251cc992 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/27dab2c7796846d1b9ff09b6251cc992 2024-11-10T17:22:42,016 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:42,021 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,021 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/cd04162849f34fa1a72e910f8cd6c4ed, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46be85c77e984dc68d8e04be7136fd0e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4d07f414a3554dc4957ee41e908ebd2b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=35.3 K 2024-11-10T17:22:42,024 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into 27dab2c7796846d1b9ff09b6251cc992(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:42,024 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:42,024 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=13, startTime=1731259361868; duration=0sec 2024-11-10T17:22:42,024 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting cd04162849f34fa1a72e910f8cd6c4ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731259360015 2024-11-10T17:22:42,025 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:42,025 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:42,032 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 46be85c77e984dc68d8e04be7136fd0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731259360393 2024-11-10T17:22:42,033 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d07f414a3554dc4957ee41e908ebd2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731259361557 2024-11-10T17:22:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:42,048 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:42,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T17:22:42,063 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:42,064 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/e869737f21ff443cacca5e689604819d is 50, key is test_row_0/C:col10/1731259361557/Put/seqid=0 2024-11-10T17:22:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T17:22:42,074 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:42,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-10T17:22:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T17:22:42,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:42,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:42,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:42,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108271636dff014208936d1c7879cea027_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259362047/Put/seqid=0 2024-11-10T17:22:42,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741993_1169 (size=12207) 2024-11-10T17:22:42,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741994_1170 (size=14594) 2024-11-10T17:22:42,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259422074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259422075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259422115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259422116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259422116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T17:22:42,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259422221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259422232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T17:22:42,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:42,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259422232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259422233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T17:22:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T17:22:42,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259422422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259422432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259422441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259422441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259422442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,501 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/e869737f21ff443cacca5e689604819d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/e869737f21ff443cacca5e689604819d 2024-11-10T17:22:42,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:42,507 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into e869737f21ff443cacca5e689604819d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:42,508 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:42,508 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=13, startTime=1731259361873; duration=0sec 2024-11-10T17:22:42,509 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:42,509 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:42,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108271636dff014208936d1c7879cea027_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108271636dff014208936d1c7879cea027_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:42,516 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/c30da92e8a494d9099009bea9a867086, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:42,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/c30da92e8a494d9099009bea9a867086 is 175, key is test_row_0/A:col10/1731259362047/Put/seqid=0 2024-11-10T17:22:42,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741995_1171 (size=39549) 2024-11-10T17:22:42,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T17:22:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T17:22:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T17:22:42,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259422741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259422752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259422757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259422757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T17:22:42,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:42,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:42,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:42,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259422932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:42,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/c30da92e8a494d9099009bea9a867086 2024-11-10T17:22:42,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/401564ca14944b5a91eae778125b8681 is 50, key is test_row_0/B:col10/1731259362047/Put/seqid=0 2024-11-10T17:22:42,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741996_1172 (size=12001) 2024-11-10T17:22:42,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/401564ca14944b5a91eae778125b8681 2024-11-10T17:22:42,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/54e22f84c2fd43a0988d21b2fa4fc8a6 is 50, key is test_row_0/C:col10/1731259362047/Put/seqid=0 2024-11-10T17:22:43,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T17:22:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:43,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:43,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741997_1173 (size=12001) 2024-11-10T17:22:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:43,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/54e22f84c2fd43a0988d21b2fa4fc8a6 2024-11-10T17:22:43,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/c30da92e8a494d9099009bea9a867086 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/c30da92e8a494d9099009bea9a867086 2024-11-10T17:22:43,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/c30da92e8a494d9099009bea9a867086, entries=200, sequenceid=123, filesize=38.6 K 2024-11-10T17:22:43,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/401564ca14944b5a91eae778125b8681 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/401564ca14944b5a91eae778125b8681 2024-11-10T17:22:43,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/401564ca14944b5a91eae778125b8681, entries=150, sequenceid=123, filesize=11.7 K 2024-11-10T17:22:43,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/54e22f84c2fd43a0988d21b2fa4fc8a6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/54e22f84c2fd43a0988d21b2fa4fc8a6 2024-11-10T17:22:43,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/54e22f84c2fd43a0988d21b2fa4fc8a6, entries=150, sequenceid=123, filesize=11.7 K 2024-11-10T17:22:43,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for afd93882106705ea245d285e044c07aa in 994ms, sequenceid=123, compaction requested=false 2024-11-10T17:22:43,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:43,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T17:22:43,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:43,161 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:22:43,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be553263aebd4d788462747d4dae0468_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259362077/Put/seqid=0 2024-11-10T17:22:43,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741998_1174 (size=12304) 2024-11-10T17:22:43,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:43,202 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be553263aebd4d788462747d4dae0468_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be553263aebd4d788462747d4dae0468_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:43,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6fd601a64e104c468f3fac6ede984404, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:43,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6fd601a64e104c468f3fac6ede984404 is 175, key is test_row_0/A:col10/1731259362077/Put/seqid=0 2024-11-10T17:22:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T17:22:43,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741999_1175 (size=31105) 2024-11-10T17:22:43,221 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6fd601a64e104c468f3fac6ede984404 2024-11-10T17:22:43,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b39f7d0c86394be9813297fc3b1bf6e9 is 50, key is test_row_0/B:col10/1731259362077/Put/seqid=0 2024-11-10T17:22:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:43,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742000_1176 (size=12151) 2024-11-10T17:22:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259423323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259423325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259423325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259423326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259423431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259423442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259423442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259423452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259423642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259423652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259423652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259423661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,695 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b39f7d0c86394be9813297fc3b1bf6e9 2024-11-10T17:22:43,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/90548315520e4b229f45b8dbcfc30f21 is 50, key is test_row_0/C:col10/1731259362077/Put/seqid=0 2024-11-10T17:22:43,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742001_1177 (size=12151) 2024-11-10T17:22:43,747 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/90548315520e4b229f45b8dbcfc30f21 2024-11-10T17:22:43,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6fd601a64e104c468f3fac6ede984404 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6fd601a64e104c468f3fac6ede984404 2024-11-10T17:22:43,762 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6fd601a64e104c468f3fac6ede984404, entries=150, sequenceid=136, filesize=30.4 K 2024-11-10T17:22:43,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b39f7d0c86394be9813297fc3b1bf6e9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b39f7d0c86394be9813297fc3b1bf6e9 2024-11-10T17:22:43,768 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b39f7d0c86394be9813297fc3b1bf6e9, entries=150, sequenceid=136, filesize=11.9 K 2024-11-10T17:22:43,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/90548315520e4b229f45b8dbcfc30f21 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/90548315520e4b229f45b8dbcfc30f21 2024-11-10T17:22:43,776 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/90548315520e4b229f45b8dbcfc30f21, entries=150, sequenceid=136, filesize=11.9 K 2024-11-10T17:22:43,781 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for afd93882106705ea245d285e044c07aa in 619ms, sequenceid=136, compaction requested=true 2024-11-10T17:22:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-10T17:22:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-10T17:22:43,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T17:22:43,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7030 sec 2024-11-10T17:22:43,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.7100 sec 2024-11-10T17:22:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:43,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:43,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:22:43,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107d2d0b33bbf94c04bd3340e5d6d8306d_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259363324/Put/seqid=0 2024-11-10T17:22:43,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259423972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259423973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259423976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742002_1178 (size=14794) 2024-11-10T17:22:43,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259423977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259423982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259424092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259424092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259424092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259424092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T17:22:44,221 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:44,222 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-10T17:22:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T17:22:44,224 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:44,224 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:44,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:44,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259424302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259424302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259424302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259424302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T17:22:44,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T17:22:44,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:44,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:44,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107d2d0b33bbf94c04bd3340e5d6d8306d_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d2d0b33bbf94c04bd3340e5d6d8306d_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:44,397 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6859906bbcb845e29a8db3e1f43e64f5, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:44,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6859906bbcb845e29a8db3e1f43e64f5 is 175, key is test_row_0/A:col10/1731259363324/Put/seqid=0 2024-11-10T17:22:44,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742003_1179 (size=39749) 2024-11-10T17:22:44,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259424482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T17:22:44,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:44,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T17:22:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259424611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259424611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259424611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259424612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:44,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T17:22:44,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:44,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,808 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6859906bbcb845e29a8db3e1f43e64f5 2024-11-10T17:22:44,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/9f53d2ebeab84f10aecaaf86135dd191 is 50, key is test_row_0/B:col10/1731259363324/Put/seqid=0 2024-11-10T17:22:44,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742004_1180 (size=12151) 2024-11-10T17:22:44,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T17:22:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T17:22:44,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T17:22:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:44,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:45,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259425132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:45,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259425132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259425132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:45,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259425133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:45,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T17:22:45,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:45,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:45,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:45,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:45,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:45,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/9f53d2ebeab84f10aecaaf86135dd191 2024-11-10T17:22:45,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/84d92304e40645999df35b8ed33cb4a1 is 50, key is test_row_0/C:col10/1731259363324/Put/seqid=0 2024-11-10T17:22:45,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742005_1181 (size=12151) 2024-11-10T17:22:45,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/84d92304e40645999df35b8ed33cb4a1 2024-11-10T17:22:45,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6859906bbcb845e29a8db3e1f43e64f5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6859906bbcb845e29a8db3e1f43e64f5 2024-11-10T17:22:45,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6859906bbcb845e29a8db3e1f43e64f5, entries=200, sequenceid=161, filesize=38.8 K 2024-11-10T17:22:45,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/9f53d2ebeab84f10aecaaf86135dd191 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/9f53d2ebeab84f10aecaaf86135dd191 2024-11-10T17:22:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/9f53d2ebeab84f10aecaaf86135dd191, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T17:22:45,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/84d92304e40645999df35b8ed33cb4a1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/84d92304e40645999df35b8ed33cb4a1 2024-11-10T17:22:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/84d92304e40645999df35b8ed33cb4a1, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T17:22:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for afd93882106705ea245d285e044c07aa in 1322ms, sequenceid=161, compaction requested=true 2024-11-10T17:22:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,278 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:45,278 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,280 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:45,280 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:45,281 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:45,281 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:45,281 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:45,281 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,281 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/27dab2c7796846d1b9ff09b6251cc992, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/c30da92e8a494d9099009bea9a867086, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6fd601a64e104c468f3fac6ede984404, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6859906bbcb845e29a8db3e1f43e64f5] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=138.2 K 2024-11-10T17:22:45,281 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d291bf7ec7a747bdb373b9f0dcb81882, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/401564ca14944b5a91eae778125b8681, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b39f7d0c86394be9813297fc3b1bf6e9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/9f53d2ebeab84f10aecaaf86135dd191] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=47.4 K 2024-11-10T17:22:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,281 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:45,281 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/27dab2c7796846d1b9ff09b6251cc992, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/c30da92e8a494d9099009bea9a867086, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6fd601a64e104c468f3fac6ede984404, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6859906bbcb845e29a8db3e1f43e64f5] 2024-11-10T17:22:45,281 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d291bf7ec7a747bdb373b9f0dcb81882, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731259361557 2024-11-10T17:22:45,281 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27dab2c7796846d1b9ff09b6251cc992, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731259361557 2024-11-10T17:22:45,282 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 401564ca14944b5a91eae778125b8681, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731259362045 2024-11-10T17:22:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,282 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30da92e8a494d9099009bea9a867086, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731259361717 2024-11-10T17:22:45,282 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b39f7d0c86394be9813297fc3b1bf6e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259362071 2024-11-10T17:22:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,283 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f53d2ebeab84f10aecaaf86135dd191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731259363294 2024-11-10T17:22:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,283 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fd601a64e104c468f3fac6ede984404, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259362071 2024-11-10T17:22:45,283 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6859906bbcb845e29a8db3e1f43e64f5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731259363294 2024-11-10T17:22:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T17:22:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:45,298 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:22:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,299 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#156 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,299 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/6e45c560b4714955ad031d27f723397c is 50, key is test_row_0/B:col10/1731259363324/Put/seqid=0 2024-11-10T17:22:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,305 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,313 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100e960bc1e8b64f6993961111fc7b16c8_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,317 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100e960bc1e8b64f6993961111fc7b16c8_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,317 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e960bc1e8b64f6993961111fc7b16c8_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742006_1182 (size=12493) 2024-11-10T17:22:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107fb12bfad5fb4fb2b1482cb1695d642c_afd93882106705ea245d285e044c07aa is 50, key is test_row_1/A:col10/1731259363967/Put/seqid=0 2024-11-10T17:22:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742008_1184 (size=9814) 2024-11-10T17:22:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742007_1183 (size=4469) 2024-11-10T17:22:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,358 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#157 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:45,359 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/2701929317184c4dae976ee1045ea9f8 is 175, key is test_row_0/A:col10/1731259363324/Put/seqid=0 2024-11-10T17:22:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T17:22:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742009_1185 (size=31447) 2024-11-10T17:22:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,376 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/2701929317184c4dae976ee1045ea9f8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2701929317184c4dae976ee1045ea9f8 2024-11-10T17:22:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,382 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into 2701929317184c4dae976ee1045ea9f8(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,382 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:45,382 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=12, startTime=1731259365278; duration=0sec 2024-11-10T17:22:45,383 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:45,383 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:45,383 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:45,384 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:45,384 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:45,384 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:45,384 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/e869737f21ff443cacca5e689604819d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/54e22f84c2fd43a0988d21b2fa4fc8a6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/90548315520e4b229f45b8dbcfc30f21, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/84d92304e40645999df35b8ed33cb4a1] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=47.4 K 2024-11-10T17:22:45,385 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e869737f21ff443cacca5e689604819d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731259361557 2024-11-10T17:22:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,385 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54e22f84c2fd43a0988d21b2fa4fc8a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731259362045 2024-11-10T17:22:45,386 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90548315520e4b229f45b8dbcfc30f21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731259362071 2024-11-10T17:22:45,386 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84d92304e40645999df35b8ed33cb4a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731259363294 2024-11-10T17:22:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,405 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:45,406 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/02f2bde3143d447cbb39241ad0454ecc is 50, key is test_row_0/C:col10/1731259363324/Put/seqid=0 2024-11-10T17:22:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742010_1186 (size=12493) 2024-11-10T17:22:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,443 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/02f2bde3143d447cbb39241ad0454ecc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/02f2bde3143d447cbb39241ad0454ecc 2024-11-10T17:22:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,451 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into 02f2bde3143d447cbb39241ad0454ecc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:45,451 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:45,451 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=12, startTime=1731259365278; duration=0sec 2024-11-10T17:22:45,452 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:45,452 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259425660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:45,730 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/6e45c560b4714955ad031d27f723397c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/6e45c560b4714955ad031d27f723397c 2024-11-10T17:22:45,736 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into 6e45c560b4714955ad031d27f723397c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:45,736 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:45,736 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=12, startTime=1731259365278; duration=0sec 2024-11-10T17:22:45,736 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:45,736 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:45,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:45,743 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107fb12bfad5fb4fb2b1482cb1695d642c_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fb12bfad5fb4fb2b1482cb1695d642c_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:45,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4e4f30b0567f4f2a9ce105de343538b2, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:45,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4e4f30b0567f4f2a9ce105de343538b2 is 175, key is test_row_1/A:col10/1731259363967/Put/seqid=0 2024-11-10T17:22:45,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742011_1187 (size=22461) 2024-11-10T17:22:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259425771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:45,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259425981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259426141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259426142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:46,145 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:46,146 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:46,151 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4e4f30b0567f4f2a9ce105de343538b2 2024-11-10T17:22:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259426152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:46,154 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:46,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/84b33755223d4bd6be129f7561662be3 is 50, key is test_row_1/B:col10/1731259363967/Put/seqid=0 2024-11-10T17:22:46,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259426161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:46,163 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:46,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742012_1188 (size=9757) 2024-11-10T17:22:46,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T17:22:46,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T17:22:46,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259426291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T17:22:46,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/84b33755223d4bd6be129f7561662be3 2024-11-10T17:22:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/dc884d7e32664b68a275aecc4a00db68 is 50, key is test_row_1/C:col10/1731259363967/Put/seqid=0 2024-11-10T17:22:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742013_1189 (size=9757) 2024-11-10T17:22:46,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259426809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:46,990 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/dc884d7e32664b68a275aecc4a00db68 2024-11-10T17:22:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4e4f30b0567f4f2a9ce105de343538b2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4e4f30b0567f4f2a9ce105de343538b2 2024-11-10T17:22:47,009 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4e4f30b0567f4f2a9ce105de343538b2, entries=100, sequenceid=173, filesize=21.9 K 2024-11-10T17:22:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/84b33755223d4bd6be129f7561662be3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/84b33755223d4bd6be129f7561662be3 2024-11-10T17:22:47,022 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/84b33755223d4bd6be129f7561662be3, entries=100, sequenceid=173, filesize=9.5 K 2024-11-10T17:22:47,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/dc884d7e32664b68a275aecc4a00db68 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/dc884d7e32664b68a275aecc4a00db68 2024-11-10T17:22:47,029 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/dc884d7e32664b68a275aecc4a00db68, entries=100, sequenceid=173, filesize=9.5 K 2024-11-10T17:22:47,030 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for afd93882106705ea245d285e044c07aa in 1732ms, sequenceid=173, compaction requested=false 2024-11-10T17:22:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-10T17:22:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-10T17:22:47,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T17:22:47,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8080 sec 2024-11-10T17:22:47,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.8130 sec 2024-11-10T17:22:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:47,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:47,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:22:47,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f9b0c467baa442839b62be2f9c5c14d2_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259367822/Put/seqid=0 2024-11-10T17:22:47,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742014_1190 (size=14794) 2024-11-10T17:22:47,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259427870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:47,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259427982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:48,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259428161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:48,166 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259428171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:48,174 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259428182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259428182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:48,184 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:48,185 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:48,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259428191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:48,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:48,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f9b0c467baa442839b62be2f9c5c14d2_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f9b0c467baa442839b62be2f9c5c14d2_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:48,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e0e820b92e5e4c37b614bfa720a00aaa, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:48,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e0e820b92e5e4c37b614bfa720a00aaa is 175, key is test_row_0/A:col10/1731259367822/Put/seqid=0 2024-11-10T17:22:48,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742015_1191 (size=39749) 2024-11-10T17:22:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T17:22:48,392 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:48,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T17:22:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T17:22:48,395 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:48,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:48,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T17:22:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259428502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:48,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T17:22:48,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:48,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:48,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:48,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e0e820b92e5e4c37b614bfa720a00aaa 2024-11-10T17:22:48,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8a8f7371b95448e28c586c2e13523510 is 50, key is test_row_0/B:col10/1731259367822/Put/seqid=0 2024-11-10T17:22:48,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742016_1192 (size=12151) 2024-11-10T17:22:48,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8a8f7371b95448e28c586c2e13523510 2024-11-10T17:22:48,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/daef474b7b274646b2af332079adbb44 is 50, key is test_row_0/C:col10/1731259367822/Put/seqid=0 2024-11-10T17:22:48,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T17:22:48,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:48,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:48,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:48,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T17:22:48,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742017_1193 (size=12151) 2024-11-10T17:22:48,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T17:22:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:48,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:49,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T17:22:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259429011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:49,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T17:22:49,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/daef474b7b274646b2af332079adbb44 2024-11-10T17:22:49,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e0e820b92e5e4c37b614bfa720a00aaa as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e0e820b92e5e4c37b614bfa720a00aaa 2024-11-10T17:22:49,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e0e820b92e5e4c37b614bfa720a00aaa, entries=200, sequenceid=201, filesize=38.8 K 2024-11-10T17:22:49,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8a8f7371b95448e28c586c2e13523510 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8a8f7371b95448e28c586c2e13523510 2024-11-10T17:22:49,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8a8f7371b95448e28c586c2e13523510, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T17:22:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/daef474b7b274646b2af332079adbb44 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/daef474b7b274646b2af332079adbb44 2024-11-10T17:22:49,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/daef474b7b274646b2af332079adbb44, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T17:22:49,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for afd93882106705ea245d285e044c07aa in 1325ms, sequenceid=201, compaction requested=true 2024-11-10T17:22:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:49,149 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:49,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:49,149 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:49,151 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:49,151 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:49,151 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,151 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:49,151 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/6e45c560b4714955ad031d27f723397c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/84b33755223d4bd6be129f7561662be3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8a8f7371b95448e28c586c2e13523510] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=33.6 K 2024-11-10T17:22:49,151 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:49,151 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,151 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2701929317184c4dae976ee1045ea9f8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4e4f30b0567f4f2a9ce105de343538b2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e0e820b92e5e4c37b614bfa720a00aaa] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=91.5 K 2024-11-10T17:22:49,151 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,152 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2701929317184c4dae976ee1045ea9f8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4e4f30b0567f4f2a9ce105de343538b2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e0e820b92e5e4c37b614bfa720a00aaa] 2024-11-10T17:22:49,152 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e45c560b4714955ad031d27f723397c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731259363294 2024-11-10T17:22:49,152 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2701929317184c4dae976ee1045ea9f8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731259363294 2024-11-10T17:22:49,152 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b33755223d4bd6be129f7561662be3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731259363967 2024-11-10T17:22:49,153 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4f30b0567f4f2a9ce105de343538b2, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731259363967 2024-11-10T17:22:49,153 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a8f7371b95448e28c586c2e13523510, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731259365654 2024-11-10T17:22:49,153 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e820b92e5e4c37b614bfa720a00aaa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731259365611 2024-11-10T17:22:49,167 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:49,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T17:22:49,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,168 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:49,170 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/58dac864dd42475cbb52371e1a25ef8b is 50, key is test_row_0/B:col10/1731259367822/Put/seqid=0 2024-11-10T17:22:49,185 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:49,209 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106768bf034ea045f4a987ed81e3dffe25_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:49,211 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106768bf034ea045f4a987ed81e3dffe25_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:49,211 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106768bf034ea045f4a987ed81e3dffe25_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100d4710eba9fb4fa1b7eb24de3170a85b_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259367835/Put/seqid=0 2024-11-10T17:22:49,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742018_1194 (size=12595) 2024-11-10T17:22:49,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742020_1196 (size=12304) 2024-11-10T17:22:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742019_1195 (size=4469) 2024-11-10T17:22:49,244 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100d4710eba9fb4fa1b7eb24de3170a85b_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100d4710eba9fb4fa1b7eb24de3170a85b_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:49,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/7b942e9d621d49a580d484476486609d, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:49,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/7b942e9d621d49a580d484476486609d is 175, key is test_row_0/A:col10/1731259367835/Put/seqid=0 2024-11-10T17:22:49,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742021_1197 (size=31105) 2024-11-10T17:22:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T17:22:49,634 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/58dac864dd42475cbb52371e1a25ef8b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/58dac864dd42475cbb52371e1a25ef8b 2024-11-10T17:22:49,643 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into 58dac864dd42475cbb52371e1a25ef8b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:49,643 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:49,643 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=13, startTime=1731259369149; duration=0sec 2024-11-10T17:22:49,643 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:49,643 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:49,643 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:49,644 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:49,644 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:49,644 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,645 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/02f2bde3143d447cbb39241ad0454ecc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/dc884d7e32664b68a275aecc4a00db68, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/daef474b7b274646b2af332079adbb44] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=33.6 K 2024-11-10T17:22:49,646 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f2bde3143d447cbb39241ad0454ecc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731259363294 2024-11-10T17:22:49,646 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#166 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:49,646 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting dc884d7e32664b68a275aecc4a00db68, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731259363967 2024-11-10T17:22:49,646 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/dee66fc6552244c4939f8705d67b0aae is 175, key is test_row_0/A:col10/1731259367822/Put/seqid=0 2024-11-10T17:22:49,648 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting daef474b7b274646b2af332079adbb44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731259365654 2024-11-10T17:22:49,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/7b942e9d621d49a580d484476486609d 2024-11-10T17:22:49,672 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:49,673 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/edf8481ffe614cc38929a8377fecb8b6 is 50, key is test_row_0/C:col10/1731259367822/Put/seqid=0 2024-11-10T17:22:49,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742022_1198 (size=31549) 2024-11-10T17:22:49,685 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/dee66fc6552244c4939f8705d67b0aae as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/dee66fc6552244c4939f8705d67b0aae 2024-11-10T17:22:49,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b4a8ee13c4594681a3f84cc9e9de4063 is 50, key is test_row_0/B:col10/1731259367835/Put/seqid=0 2024-11-10T17:22:49,692 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into dee66fc6552244c4939f8705d67b0aae(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:49,692 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:49,692 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=13, startTime=1731259369149; duration=0sec 2024-11-10T17:22:49,693 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:49,693 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:49,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742023_1199 (size=12595) 2024-11-10T17:22:49,714 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/edf8481ffe614cc38929a8377fecb8b6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/edf8481ffe614cc38929a8377fecb8b6 2024-11-10T17:22:49,720 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into edf8481ffe614cc38929a8377fecb8b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:49,720 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:49,720 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=13, startTime=1731259369149; duration=0sec 2024-11-10T17:22:49,720 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:49,720 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:49,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742024_1200 (size=12151) 2024-11-10T17:22:49,743 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b4a8ee13c4594681a3f84cc9e9de4063 2024-11-10T17:22:49,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/c3ff9181c87d49a9a0a28d934414890f is 50, key is test_row_0/C:col10/1731259367835/Put/seqid=0 2024-11-10T17:22:49,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742025_1201 (size=12151) 2024-11-10T17:22:49,765 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/c3ff9181c87d49a9a0a28d934414890f 2024-11-10T17:22:49,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/7b942e9d621d49a580d484476486609d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/7b942e9d621d49a580d484476486609d 2024-11-10T17:22:49,786 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/7b942e9d621d49a580d484476486609d, entries=150, sequenceid=212, filesize=30.4 K 2024-11-10T17:22:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b4a8ee13c4594681a3f84cc9e9de4063 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b4a8ee13c4594681a3f84cc9e9de4063 2024-11-10T17:22:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,796 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b4a8ee13c4594681a3f84cc9e9de4063, entries=150, sequenceid=212, filesize=11.9 K 2024-11-10T17:22:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/c3ff9181c87d49a9a0a28d934414890f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/c3ff9181c87d49a9a0a28d934414890f 2024-11-10T17:22:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,804 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/c3ff9181c87d49a9a0a28d934414890f, entries=150, sequenceid=212, filesize=11.9 K 2024-11-10T17:22:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,805 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for afd93882106705ea245d285e044c07aa in 637ms, sequenceid=212, compaction requested=false 2024-11-10T17:22:49,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:49,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:49,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T17:22:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T17:22:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T17:22:49,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-10T17:22:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4180 sec 2024-11-10T17:22:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:50,171 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:50,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:22:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102726a1558fca4aec87cc43e59ff663cc_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259370138/Put/seqid=0 2024-11-10T17:22:50,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742026_1202 (size=22268) 2024-11-10T17:22:50,217 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,225 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102726a1558fca4aec87cc43e59ff663cc_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102726a1558fca4aec87cc43e59ff663cc_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:50,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/9651cf9a836943ddb2361a8543aa3fe5, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:50,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/9651cf9a836943ddb2361a8543aa3fe5 is 175, key is test_row_0/A:col10/1731259370138/Put/seqid=0 2024-11-10T17:22:50,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742027_1203 (size=65673) 2024-11-10T17:22:50,238 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/9651cf9a836943ddb2361a8543aa3fe5 2024-11-10T17:22:50,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/c9a71878d3fe487eb37c4be00595c116 is 50, key is test_row_0/B:col10/1731259370138/Put/seqid=0 2024-11-10T17:22:50,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742028_1204 (size=12151) 2024-11-10T17:22:50,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/c9a71878d3fe487eb37c4be00595c116 2024-11-10T17:22:50,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/9a9b420d6191417da43ef7ce408f3e22 is 50, key is test_row_0/C:col10/1731259370138/Put/seqid=0 2024-11-10T17:22:50,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742029_1205 (size=12151) 2024-11-10T17:22:50,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/9a9b420d6191417da43ef7ce408f3e22 2024-11-10T17:22:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/9651cf9a836943ddb2361a8543aa3fe5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9651cf9a836943ddb2361a8543aa3fe5 2024-11-10T17:22:50,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9651cf9a836943ddb2361a8543aa3fe5, entries=350, sequenceid=226, filesize=64.1 K 2024-11-10T17:22:50,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/c9a71878d3fe487eb37c4be00595c116 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c9a71878d3fe487eb37c4be00595c116 2024-11-10T17:22:50,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c9a71878d3fe487eb37c4be00595c116, entries=150, sequenceid=226, filesize=11.9 K 2024-11-10T17:22:50,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/9a9b420d6191417da43ef7ce408f3e22 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9a9b420d6191417da43ef7ce408f3e22 2024-11-10T17:22:50,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9a9b420d6191417da43ef7ce408f3e22, entries=150, sequenceid=226, filesize=11.9 K 2024-11-10T17:22:50,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=140.89 KB/144270 for afd93882106705ea245d285e044c07aa in 169ms, sequenceid=226, compaction requested=true 2024-11-10T17:22:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:50,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:50,341 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:50,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:50,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:50,342 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128327 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:50,343 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:50,343 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,343 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/dee66fc6552244c4939f8705d67b0aae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/7b942e9d621d49a580d484476486609d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9651cf9a836943ddb2361a8543aa3fe5] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=125.3 K 2024-11-10T17:22:50,343 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,343 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/dee66fc6552244c4939f8705d67b0aae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/7b942e9d621d49a580d484476486609d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9651cf9a836943ddb2361a8543aa3fe5] 2024-11-10T17:22:50,343 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting dee66fc6552244c4939f8705d67b0aae, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731259365654 2024-11-10T17:22:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:50,344 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:50,344 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b942e9d621d49a580d484476486609d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731259367835 2024-11-10T17:22:50,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:50,344 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9651cf9a836943ddb2361a8543aa3fe5, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731259370100 2024-11-10T17:22:50,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:50,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:50,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:50,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:22:50,346 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:50,346 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:50,346 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,346 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/58dac864dd42475cbb52371e1a25ef8b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b4a8ee13c4594681a3f84cc9e9de4063, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c9a71878d3fe487eb37c4be00595c116] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=36.0 K 2024-11-10T17:22:50,346 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 58dac864dd42475cbb52371e1a25ef8b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731259365654 2024-11-10T17:22:50,349 INFO [master/8e3de9160f9f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T17:22:50,349 INFO [master/8e3de9160f9f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T17:22:50,349 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b4a8ee13c4594681a3f84cc9e9de4063, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731259367835 2024-11-10T17:22:50,350 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c9a71878d3fe487eb37c4be00595c116, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731259370138 2024-11-10T17:22:50,366 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:50,379 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:50,380 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/f0f2955f554f4fef9979c6fbcbd88cac is 50, key is test_row_0/B:col10/1731259370138/Put/seqid=0 2024-11-10T17:22:50,393 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110cc875520312a40269486e6852e922629_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:50,396 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110cc875520312a40269486e6852e922629_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:50,396 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cc875520312a40269486e6852e922629_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259430398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:50,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106185b9817aa74c73a87bd3461e6b1cc5_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259370330/Put/seqid=0 2024-11-10T17:22:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742030_1206 (size=12697) 2024-11-10T17:22:50,450 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/f0f2955f554f4fef9979c6fbcbd88cac as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f0f2955f554f4fef9979c6fbcbd88cac 2024-11-10T17:22:50,468 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into f0f2955f554f4fef9979c6fbcbd88cac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:50,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:50,468 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=13, startTime=1731259370343; duration=0sec 2024-11-10T17:22:50,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:50,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:50,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:50,470 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:50,470 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:50,470 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,470 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/edf8481ffe614cc38929a8377fecb8b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/c3ff9181c87d49a9a0a28d934414890f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9a9b420d6191417da43ef7ce408f3e22] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=36.0 K 2024-11-10T17:22:50,471 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting edf8481ffe614cc38929a8377fecb8b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731259365654 2024-11-10T17:22:50,471 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ff9181c87d49a9a0a28d934414890f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731259367835 2024-11-10T17:22:50,472 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a9b420d6191417da43ef7ce408f3e22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731259370138 2024-11-10T17:22:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742031_1207 (size=4469) 2024-11-10T17:22:50,490 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:50,491 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/b5833fbc89c749cea2d5a1aa0265847e is 50, key is test_row_0/C:col10/1731259370138/Put/seqid=0 2024-11-10T17:22:50,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742032_1208 (size=12304) 2024-11-10T17:22:50,507 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:50,512 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106185b9817aa74c73a87bd3461e6b1cc5_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106185b9817aa74c73a87bd3461e6b1cc5_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259430513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:50,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4d3d56af65da48409eb97730a1e36478, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:50,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4d3d56af65da48409eb97730a1e36478 is 175, key is test_row_0/A:col10/1731259370330/Put/seqid=0 2024-11-10T17:22:50,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742033_1209 (size=12697) 2024-11-10T17:22:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T17:22:50,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742034_1210 (size=31105) 2024-11-10T17:22:50,542 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:50,543 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4d3d56af65da48409eb97730a1e36478 2024-11-10T17:22:50,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T17:22:50,545 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T17:22:50,547 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:50,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:50,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8b82008f2cd0402c875e6b5c247f5ce3 is 50, key is test_row_0/B:col10/1731259370330/Put/seqid=0 2024-11-10T17:22:50,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742035_1211 (size=12151) 2024-11-10T17:22:50,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8b82008f2cd0402c875e6b5c247f5ce3 2024-11-10T17:22:50,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/4155722b8dc04da28e1caa7171a86eca is 50, key is test_row_0/C:col10/1731259370330/Put/seqid=0 2024-11-10T17:22:50,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742036_1212 (size=12151) 2024-11-10T17:22:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T17:22:50,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:50,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:50,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:50,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259430721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:50,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:50,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T17:22:50,888 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#174 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:50,889 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/d6dcdce09545468db98ba646b6af6dd6 is 175, key is test_row_0/A:col10/1731259370138/Put/seqid=0 2024-11-10T17:22:50,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742037_1213 (size=31651) 2024-11-10T17:22:50,915 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/d6dcdce09545468db98ba646b6af6dd6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/d6dcdce09545468db98ba646b6af6dd6 2024-11-10T17:22:50,921 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into d6dcdce09545468db98ba646b6af6dd6(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:50,921 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:50,921 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=13, startTime=1731259370341; duration=0sec 2024-11-10T17:22:50,922 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:50,922 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:50,944 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/b5833fbc89c749cea2d5a1aa0265847e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b5833fbc89c749cea2d5a1aa0265847e 2024-11-10T17:22:50,951 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into b5833fbc89c749cea2d5a1aa0265847e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:50,951 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:50,951 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=13, startTime=1731259370345; duration=0sec 2024-11-10T17:22:50,951 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:50,951 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:50,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/4155722b8dc04da28e1caa7171a86eca 2024-11-10T17:22:51,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4d3d56af65da48409eb97730a1e36478 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d3d56af65da48409eb97730a1e36478 2024-11-10T17:22:51,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d3d56af65da48409eb97730a1e36478, entries=150, sequenceid=251, filesize=30.4 K 2024-11-10T17:22:51,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8b82008f2cd0402c875e6b5c247f5ce3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8b82008f2cd0402c875e6b5c247f5ce3 2024-11-10T17:22:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8b82008f2cd0402c875e6b5c247f5ce3, entries=150, sequenceid=251, filesize=11.9 K 2024-11-10T17:22:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/4155722b8dc04da28e1caa7171a86eca as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4155722b8dc04da28e1caa7171a86eca 2024-11-10T17:22:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4155722b8dc04da28e1caa7171a86eca, entries=150, sequenceid=251, filesize=11.9 K 2024-11-10T17:22:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for afd93882106705ea245d285e044c07aa in 677ms, sequenceid=251, compaction requested=false 2024-11-10T17:22:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:51,042 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:51,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:22:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107c9b9d7cf81643869ccfdcc850b8b067_afd93882106705ea245d285e044c07aa is 50, key is test_row_1/A:col10/1731259370392/Put/seqid=0 2024-11-10T17:22:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742038_1214 (size=12454) 2024-11-10T17:22:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:51,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:51,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T17:22:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259431173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259431282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:51,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:51,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:51,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:51,491 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107c9b9d7cf81643869ccfdcc850b8b067_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107c9b9d7cf81643869ccfdcc850b8b067_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:51,492 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/2a38e3959f764a9e88124470746bfe65, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:51,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:51,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/2a38e3959f764a9e88124470746bfe65 is 175, key is test_row_1/A:col10/1731259370392/Put/seqid=0 2024-11-10T17:22:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259431492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742039_1215 (size=31251) 2024-11-10T17:22:51,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/2a38e3959f764a9e88124470746bfe65 2024-11-10T17:22:51,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8181ef62fbbd4a4fb6b5e2391aac922a is 50, key is test_row_1/B:col10/1731259370392/Put/seqid=0 2024-11-10T17:22:51,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742040_1216 (size=9857) 2024-11-10T17:22:51,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8181ef62fbbd4a4fb6b5e2391aac922a 2024-11-10T17:22:51,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/665e4afddd2146ee83ede4b39e948d03 is 50, key is test_row_1/C:col10/1731259370392/Put/seqid=0 2024-11-10T17:22:51,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742041_1217 (size=9857) 2024-11-10T17:22:51,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T17:22:51,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259431801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:51,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:51,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/665e4afddd2146ee83ede4b39e948d03 2024-11-10T17:22:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/2a38e3959f764a9e88124470746bfe65 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2a38e3959f764a9e88124470746bfe65 2024-11-10T17:22:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2a38e3959f764a9e88124470746bfe65, entries=150, sequenceid=266, filesize=30.5 K 2024-11-10T17:22:51,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8181ef62fbbd4a4fb6b5e2391aac922a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8181ef62fbbd4a4fb6b5e2391aac922a 2024-11-10T17:22:51,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8181ef62fbbd4a4fb6b5e2391aac922a, entries=100, sequenceid=266, filesize=9.6 K 2024-11-10T17:22:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/665e4afddd2146ee83ede4b39e948d03 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/665e4afddd2146ee83ede4b39e948d03 2024-11-10T17:22:51,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/665e4afddd2146ee83ede4b39e948d03, entries=100, sequenceid=266, filesize=9.6 K 2024-11-10T17:22:51,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for afd93882106705ea245d285e044c07aa in 932ms, sequenceid=266, compaction requested=true 2024-11-10T17:22:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:51,974 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:51,974 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:51,975 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:51,975 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:51,975 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:51,975 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:51,975 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,975 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,975 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f0f2955f554f4fef9979c6fbcbd88cac, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8b82008f2cd0402c875e6b5c247f5ce3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8181ef62fbbd4a4fb6b5e2391aac922a] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=33.9 K 2024-11-10T17:22:51,975 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/d6dcdce09545468db98ba646b6af6dd6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d3d56af65da48409eb97730a1e36478, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2a38e3959f764a9e88124470746bfe65] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=91.8 K 2024-11-10T17:22:51,976 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:51,976 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/d6dcdce09545468db98ba646b6af6dd6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d3d56af65da48409eb97730a1e36478, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2a38e3959f764a9e88124470746bfe65] 2024-11-10T17:22:51,976 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f0f2955f554f4fef9979c6fbcbd88cac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731259370138 2024-11-10T17:22:51,976 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6dcdce09545468db98ba646b6af6dd6, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731259370138 2024-11-10T17:22:51,976 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b82008f2cd0402c875e6b5c247f5ce3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731259370314 2024-11-10T17:22:51,976 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3d56af65da48409eb97730a1e36478, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731259370314 2024-11-10T17:22:51,977 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8181ef62fbbd4a4fb6b5e2391aac922a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731259370392 2024-11-10T17:22:51,977 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a38e3959f764a9e88124470746bfe65, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731259370384 2024-11-10T17:22:51,993 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:52,001 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411104465e5f340284b22b9f32452f66e5616_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:52,003 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:52,003 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411104465e5f340284b22b9f32452f66e5616_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:52,003 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/60ff5e0c74124a1685ba1e8f6af45e9b is 50, key is test_row_0/B:col10/1731259370330/Put/seqid=0 2024-11-10T17:22:52,003 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104465e5f340284b22b9f32452f66e5616_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:52,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742042_1218 (size=12899) 2024-11-10T17:22:52,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742043_1219 (size=4469) 2024-11-10T17:22:52,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T17:22:52,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:52,082 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:22:52,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a35f3be9ca0747d2ad896004078610d9_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259371171/Put/seqid=0 2024-11-10T17:22:52,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742044_1220 (size=12454) 2024-11-10T17:22:52,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:52,110 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a35f3be9ca0747d2ad896004078610d9_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a35f3be9ca0747d2ad896004078610d9_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:52,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/ec7b7863f7cb47b093fa41e24e60b11f, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/ec7b7863f7cb47b093fa41e24e60b11f is 175, key is test_row_0/A:col10/1731259371171/Put/seqid=0 2024-11-10T17:22:52,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742045_1221 (size=31255) 2024-11-10T17:22:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:52,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:52,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259432201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51662 deadline: 1731259432201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259432202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,207 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51628 deadline: 1731259432222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,223 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in afd93882106705ea245d285e044c07aa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:22:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259432311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259432312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259432312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,411 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:52,412 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4848c466b9fd4e36b345a1b03537062f is 175, key is test_row_0/A:col10/1731259370330/Put/seqid=0 2024-11-10T17:22:52,417 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/60ff5e0c74124a1685ba1e8f6af45e9b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/60ff5e0c74124a1685ba1e8f6af45e9b 2024-11-10T17:22:52,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742046_1222 (size=31960) 2024-11-10T17:22:52,423 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into 60ff5e0c74124a1685ba1e8f6af45e9b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:52,423 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:52,423 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=13, startTime=1731259371974; duration=0sec 2024-11-10T17:22:52,424 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:52,424 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:52,424 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:52,427 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:52,427 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:52,427 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:52,427 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b5833fbc89c749cea2d5a1aa0265847e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4155722b8dc04da28e1caa7171a86eca, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/665e4afddd2146ee83ede4b39e948d03] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=33.9 K 2024-11-10T17:22:52,427 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b5833fbc89c749cea2d5a1aa0265847e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731259370138 2024-11-10T17:22:52,428 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4155722b8dc04da28e1caa7171a86eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731259370314 2024-11-10T17:22:52,428 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 665e4afddd2146ee83ede4b39e948d03, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731259370392 2024-11-10T17:22:52,436 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:52,437 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/6af3d06de601411e9733ee81250a1f64 is 50, key is test_row_0/C:col10/1731259370330/Put/seqid=0 2024-11-10T17:22:52,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742047_1223 (size=12899) 2024-11-10T17:22:52,464 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/6af3d06de601411e9733ee81250a1f64 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/6af3d06de601411e9733ee81250a1f64 2024-11-10T17:22:52,470 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into 6af3d06de601411e9733ee81250a1f64(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:52,470 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:52,470 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=13, startTime=1731259371974; duration=0sec 2024-11-10T17:22:52,471 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:52,471 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:52,516 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/ec7b7863f7cb47b093fa41e24e60b11f 2024-11-10T17:22:52,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259432522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259432522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/831fc4b9a8b84e0982008f58169a762d is 50, key is test_row_0/B:col10/1731259371171/Put/seqid=0 2024-11-10T17:22:52,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742048_1224 (size=12301) 2024-11-10T17:22:52,536 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/831fc4b9a8b84e0982008f58169a762d 2024-11-10T17:22:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/513e817797b649e1856a4eeaddb4b745 is 50, key is test_row_0/C:col10/1731259371171/Put/seqid=0 2024-11-10T17:22:52,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742049_1225 (size=12301) 2024-11-10T17:22:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T17:22:52,829 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/4848c466b9fd4e36b345a1b03537062f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4848c466b9fd4e36b345a1b03537062f 2024-11-10T17:22:52,834 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into 4848c466b9fd4e36b345a1b03537062f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:52,834 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:52,834 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=13, startTime=1731259371974; duration=0sec 2024-11-10T17:22:52,834 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:52,834 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:52,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259432831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259432831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/513e817797b649e1856a4eeaddb4b745 2024-11-10T17:22:52,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/ec7b7863f7cb47b093fa41e24e60b11f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/ec7b7863f7cb47b093fa41e24e60b11f 2024-11-10T17:22:52,974 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/ec7b7863f7cb47b093fa41e24e60b11f, entries=150, sequenceid=290, filesize=30.5 K 2024-11-10T17:22:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/831fc4b9a8b84e0982008f58169a762d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/831fc4b9a8b84e0982008f58169a762d 2024-11-10T17:22:52,980 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/831fc4b9a8b84e0982008f58169a762d, entries=150, sequenceid=290, filesize=12.0 K 2024-11-10T17:22:52,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/513e817797b649e1856a4eeaddb4b745 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/513e817797b649e1856a4eeaddb4b745 2024-11-10T17:22:52,986 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/513e817797b649e1856a4eeaddb4b745, entries=150, sequenceid=290, filesize=12.0 K 2024-11-10T17:22:52,987 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for afd93882106705ea245d285e044c07aa in 904ms, sequenceid=290, compaction requested=false 2024-11-10T17:22:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T17:22:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T17:22:52,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T17:22:52,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4410 sec 2024-11-10T17:22:52,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4470 sec 2024-11-10T17:22:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:53,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:22:53,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106c9f3e991d1c42449fd2b9cc611a20d6_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259372195/Put/seqid=0 2024-11-10T17:22:53,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742050_1226 (size=12454) 2024-11-10T17:22:53,352 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:53,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106c9f3e991d1c42449fd2b9cc611a20d6_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106c9f3e991d1c42449fd2b9cc611a20d6_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:53,367 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/3eb228aa6d3944c98b7be369f5e02ad5, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:53,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/3eb228aa6d3944c98b7be369f5e02ad5 is 175, key is test_row_0/A:col10/1731259372195/Put/seqid=0 2024-11-10T17:22:53,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742051_1227 (size=31255) 2024-11-10T17:22:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259433412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259433412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259433414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259433521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259433521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259433532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259433731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259433731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259433742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:53,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:53,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:53,777 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/3eb228aa6d3944c98b7be369f5e02ad5 2024-11-10T17:22:53,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ec3ac21504e74e37841039acab38ec88 is 50, key is test_row_0/B:col10/1731259372195/Put/seqid=0 2024-11-10T17:22:53,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742052_1228 (size=12301) 2024-11-10T17:22:53,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ec3ac21504e74e37841039acab38ec88 2024-11-10T17:22:53,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/d72bcdcf482c4cbbb9330ef37b0d9402 is 50, key is test_row_0/C:col10/1731259372195/Put/seqid=0 2024-11-10T17:22:53,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742053_1229 (size=12301) 2024-11-10T17:22:54,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259434042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259434042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259434051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/d72bcdcf482c4cbbb9330ef37b0d9402 2024-11-10T17:22:54,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/3eb228aa6d3944c98b7be369f5e02ad5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/3eb228aa6d3944c98b7be369f5e02ad5 2024-11-10T17:22:54,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/3eb228aa6d3944c98b7be369f5e02ad5, entries=150, sequenceid=306, filesize=30.5 K 2024-11-10T17:22:54,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/ec3ac21504e74e37841039acab38ec88 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ec3ac21504e74e37841039acab38ec88 2024-11-10T17:22:54,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ec3ac21504e74e37841039acab38ec88, entries=150, sequenceid=306, filesize=12.0 K 2024-11-10T17:22:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/d72bcdcf482c4cbbb9330ef37b0d9402 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d72bcdcf482c4cbbb9330ef37b0d9402 2024-11-10T17:22:54,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d72bcdcf482c4cbbb9330ef37b0d9402, entries=150, sequenceid=306, filesize=12.0 K 2024-11-10T17:22:54,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for afd93882106705ea245d285e044c07aa in 923ms, sequenceid=306, compaction requested=true 2024-11-10T17:22:54,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:54,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:54,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:54,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:54,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T17:22:54,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:54,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T17:22:54,249 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:54,249 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:54,250 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94470 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:54,250 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:54,250 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:54,250 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4848c466b9fd4e36b345a1b03537062f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/ec7b7863f7cb47b093fa41e24e60b11f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/3eb228aa6d3944c98b7be369f5e02ad5] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=92.3 K 2024-11-10T17:22:54,250 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:54,250 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4848c466b9fd4e36b345a1b03537062f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/ec7b7863f7cb47b093fa41e24e60b11f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/3eb228aa6d3944c98b7be369f5e02ad5] 2024-11-10T17:22:54,251 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4848c466b9fd4e36b345a1b03537062f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731259370330 2024-11-10T17:22:54,251 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7b7863f7cb47b093fa41e24e60b11f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731259371155 2024-11-10T17:22:54,251 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:54,252 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:54,252 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:54,252 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/6af3d06de601411e9733ee81250a1f64, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/513e817797b649e1856a4eeaddb4b745, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d72bcdcf482c4cbbb9330ef37b0d9402] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=36.6 K 2024-11-10T17:22:54,252 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb228aa6d3944c98b7be369f5e02ad5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731259372195 2024-11-10T17:22:54,253 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6af3d06de601411e9733ee81250a1f64, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731259370330 2024-11-10T17:22:54,254 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 513e817797b649e1856a4eeaddb4b745, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731259371155 2024-11-10T17:22:54,254 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d72bcdcf482c4cbbb9330ef37b0d9402, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731259372195 2024-11-10T17:22:54,265 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:54,277 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411105fbc65d04f3447058d8258607019eac5_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:54,279 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411105fbc65d04f3447058d8258607019eac5_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:54,279 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105fbc65d04f3447058d8258607019eac5_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:54,282 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:54,283 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/a7f4789b8ad949d2a2d2584cce35f12b is 50, key is test_row_0/C:col10/1731259372195/Put/seqid=0 2024-11-10T17:22:54,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742054_1230 (size=4469) 2024-11-10T17:22:54,309 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#192 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:54,310 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/9f2a55754b604904af5c1c7c3d781dbd is 175, key is test_row_0/A:col10/1731259372195/Put/seqid=0 2024-11-10T17:22:54,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742055_1231 (size=13051) 2024-11-10T17:22:54,337 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/a7f4789b8ad949d2a2d2584cce35f12b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a7f4789b8ad949d2a2d2584cce35f12b 2024-11-10T17:22:54,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742056_1232 (size=32005) 2024-11-10T17:22:54,350 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into a7f4789b8ad949d2a2d2584cce35f12b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:54,350 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:54,350 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=13, startTime=1731259374249; duration=0sec 2024-11-10T17:22:54,350 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:54,350 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:54,351 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:22:54,353 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:22:54,353 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:54,353 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:54,354 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/60ff5e0c74124a1685ba1e8f6af45e9b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/831fc4b9a8b84e0982008f58169a762d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ec3ac21504e74e37841039acab38ec88] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=36.6 K 2024-11-10T17:22:54,354 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ff5e0c74124a1685ba1e8f6af45e9b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731259370330 2024-11-10T17:22:54,354 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 831fc4b9a8b84e0982008f58169a762d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731259371155 2024-11-10T17:22:54,355 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting ec3ac21504e74e37841039acab38ec88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731259372195 2024-11-10T17:22:54,355 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/9f2a55754b604904af5c1c7c3d781dbd as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9f2a55754b604904af5c1c7c3d781dbd 2024-11-10T17:22:54,362 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into 9f2a55754b604904af5c1c7c3d781dbd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:54,362 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:54,363 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=13, startTime=1731259374248; duration=0sec 2024-11-10T17:22:54,363 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:54,363 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:54,374 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:54,375 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8400f5fa4706496bb7661d94b371f5f1 is 50, key is test_row_0/B:col10/1731259372195/Put/seqid=0 2024-11-10T17:22:54,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742057_1233 (size=13051) 2024-11-10T17:22:54,400 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8400f5fa4706496bb7661d94b371f5f1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8400f5fa4706496bb7661d94b371f5f1 2024-11-10T17:22:54,408 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into 8400f5fa4706496bb7661d94b371f5f1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:54,409 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:54,409 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=13, startTime=1731259374248; duration=0sec 2024-11-10T17:22:54,409 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:54,409 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:54,554 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:54,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:22:54,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bea292926de6462da70ba2829229a1c0_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259374554/Put/seqid=0 2024-11-10T17:22:54,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742058_1234 (size=14994) 2024-11-10T17:22:54,570 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:54,573 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bea292926de6462da70ba2829229a1c0_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bea292926de6462da70ba2829229a1c0_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:54,574 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/785b2965f20c470ebc40a8b7400fac96, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:54,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/785b2965f20c470ebc40a8b7400fac96 is 175, key is test_row_0/A:col10/1731259374554/Put/seqid=0 2024-11-10T17:22:54,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742059_1235 (size=39949) 2024-11-10T17:22:54,581 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/785b2965f20c470ebc40a8b7400fac96 2024-11-10T17:22:54,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d87190153ab94c7792ae15f66a0289c9 is 50, key is test_row_0/B:col10/1731259374554/Put/seqid=0 2024-11-10T17:22:54,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742060_1236 (size=12301) 2024-11-10T17:22:54,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259434606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259434607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259434612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T17:22:54,712 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:54,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T17:22:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T17:22:54,714 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:54,715 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:54,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:54,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259434722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259434722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259434722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T17:22:54,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T17:22:54,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:54,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:54,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:54,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:54,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:54,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259434932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259434932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259434932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:54,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d87190153ab94c7792ae15f66a0289c9 2024-11-10T17:22:55,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/a2e38c3cd7de46b38b7792dfce737a69 is 50, key is test_row_0/C:col10/1731259374554/Put/seqid=0 2024-11-10T17:22:55,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742061_1237 (size=12301) 2024-11-10T17:22:55,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T17:22:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:55,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T17:22:55,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T17:22:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:55,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259435241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259435241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259435242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T17:22:55,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:55,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:55,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:55,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T17:22:55,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/a2e38c3cd7de46b38b7792dfce737a69 2024-11-10T17:22:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/785b2965f20c470ebc40a8b7400fac96 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/785b2965f20c470ebc40a8b7400fac96 2024-11-10T17:22:55,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/785b2965f20c470ebc40a8b7400fac96, entries=200, sequenceid=335, filesize=39.0 K 2024-11-10T17:22:55,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/d87190153ab94c7792ae15f66a0289c9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d87190153ab94c7792ae15f66a0289c9 2024-11-10T17:22:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d87190153ab94c7792ae15f66a0289c9, entries=150, sequenceid=335, filesize=12.0 K 2024-11-10T17:22:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/a2e38c3cd7de46b38b7792dfce737a69 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a2e38c3cd7de46b38b7792dfce737a69 2024-11-10T17:22:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a2e38c3cd7de46b38b7792dfce737a69, entries=150, sequenceid=335, filesize=12.0 K 2024-11-10T17:22:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for afd93882106705ea245d285e044c07aa in 896ms, sequenceid=335, compaction requested=false 2024-11-10T17:22:55,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T17:22:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:55,484 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T17:22:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e2550cd046343a7a9b03dc6144f1b78_afd93882106705ea245d285e044c07aa is 50, key is test_row_1/A:col10/1731259374568/Put/seqid=0 2024-11-10T17:22:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742062_1238 (size=9914) 2024-11-10T17:22:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,504 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e2550cd046343a7a9b03dc6144f1b78_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e2550cd046343a7a9b03dc6144f1b78_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aad5494ce8514a10a025a154d8c5ff34, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aad5494ce8514a10a025a154d8c5ff34 is 175, key is test_row_1/A:col10/1731259374568/Put/seqid=0 2024-11-10T17:22:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742063_1239 (size=22561) 2024-11-10T17:22:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:55,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259435837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259435837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259435838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T17:22:55,940 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aad5494ce8514a10a025a154d8c5ff34 2024-11-10T17:22:55,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8dc5de7c96834eca95fae88f36c4742e is 50, key is test_row_1/B:col10/1731259374568/Put/seqid=0 2024-11-10T17:22:55,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259435966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259435966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259435966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:55,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742064_1240 (size=9857) 2024-11-10T17:22:56,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259436181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259436182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259436182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,387 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8dc5de7c96834eca95fae88f36c4742e 2024-11-10T17:22:56,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/821e4a19b2fd434488d79c2acecceb7c is 50, key is test_row_1/C:col10/1731259374568/Put/seqid=0 2024-11-10T17:22:56,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742065_1241 (size=9857) 2024-11-10T17:22:56,406 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/821e4a19b2fd434488d79c2acecceb7c 2024-11-10T17:22:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/aad5494ce8514a10a025a154d8c5ff34 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aad5494ce8514a10a025a154d8c5ff34 2024-11-10T17:22:56,419 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aad5494ce8514a10a025a154d8c5ff34, entries=100, sequenceid=345, filesize=22.0 K 2024-11-10T17:22:56,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/8dc5de7c96834eca95fae88f36c4742e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8dc5de7c96834eca95fae88f36c4742e 2024-11-10T17:22:56,426 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8dc5de7c96834eca95fae88f36c4742e, entries=100, sequenceid=345, filesize=9.6 K 2024-11-10T17:22:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/821e4a19b2fd434488d79c2acecceb7c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/821e4a19b2fd434488d79c2acecceb7c 2024-11-10T17:22:56,440 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/821e4a19b2fd434488d79c2acecceb7c, entries=100, sequenceid=345, filesize=9.6 K 2024-11-10T17:22:56,440 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for afd93882106705ea245d285e044c07aa in 957ms, sequenceid=345, compaction requested=true 2024-11-10T17:22:56,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:56,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:56,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T17:22:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T17:22:56,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T17:22:56,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7270 sec 2024-11-10T17:22:56,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7350 sec 2024-11-10T17:22:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:56,496 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:56,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T17:22:56,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b45400a3036f4732b9271a25aaacbfde_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259376495/Put/seqid=0 2024-11-10T17:22:56,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259436503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259436509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259436510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742066_1242 (size=20074) 2024-11-10T17:22:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259436621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259436621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259436621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259436832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259436832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259436832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T17:22:56,862 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:56,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:22:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T17:22:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T17:22:56,864 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:22:56,865 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:22:56,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:22:56,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:56,928 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b45400a3036f4732b9271a25aaacbfde_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b45400a3036f4732b9271a25aaacbfde_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:56,929 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/8a56009445334a55b5d214cc04dd38a1, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:56,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/8a56009445334a55b5d214cc04dd38a1 is 175, key is test_row_0/A:col10/1731259376495/Put/seqid=0 2024-11-10T17:22:56,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742067_1243 (size=57333) 2024-11-10T17:22:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T17:22:57,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T17:22:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259437141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259437141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259437141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T17:22:57,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:57,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T17:22:57,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T17:22:57,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:57,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:22:57,349 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/8a56009445334a55b5d214cc04dd38a1 2024-11-10T17:22:57,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b38333e23eec44f0ae544d76c22684fc is 50, key is test_row_0/B:col10/1731259376495/Put/seqid=0 2024-11-10T17:22:57,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742068_1244 (size=12301) 2024-11-10T17:22:57,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b38333e23eec44f0ae544d76c22684fc 2024-11-10T17:22:57,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/deb21a9bb25448da86362b94b3153dfb is 50, key is test_row_0/C:col10/1731259376495/Put/seqid=0 2024-11-10T17:22:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742069_1245 (size=12301) 2024-11-10T17:22:57,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/deb21a9bb25448da86362b94b3153dfb 2024-11-10T17:22:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/8a56009445334a55b5d214cc04dd38a1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8a56009445334a55b5d214cc04dd38a1 2024-11-10T17:22:57,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8a56009445334a55b5d214cc04dd38a1, entries=300, sequenceid=374, filesize=56.0 K 2024-11-10T17:22:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/b38333e23eec44f0ae544d76c22684fc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b38333e23eec44f0ae544d76c22684fc 2024-11-10T17:22:57,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b38333e23eec44f0ae544d76c22684fc, entries=150, sequenceid=374, filesize=12.0 K 2024-11-10T17:22:57,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/deb21a9bb25448da86362b94b3153dfb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/deb21a9bb25448da86362b94b3153dfb 2024-11-10T17:22:57,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/deb21a9bb25448da86362b94b3153dfb, entries=150, sequenceid=374, filesize=12.0 K 2024-11-10T17:22:57,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for afd93882106705ea245d285e044c07aa in 918ms, sequenceid=374, compaction requested=true 2024-11-10T17:22:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:22:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:22:57,414 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:22:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T17:22:57,416 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151848 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:57,416 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:22:57,416 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,416 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9f2a55754b604904af5c1c7c3d781dbd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/785b2965f20c470ebc40a8b7400fac96, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aad5494ce8514a10a025a154d8c5ff34, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8a56009445334a55b5d214cc04dd38a1] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=148.3 K 2024-11-10T17:22:57,416 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,416 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9f2a55754b604904af5c1c7c3d781dbd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/785b2965f20c470ebc40a8b7400fac96, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aad5494ce8514a10a025a154d8c5ff34, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8a56009445334a55b5d214cc04dd38a1] 2024-11-10T17:22:57,417 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2a55754b604904af5c1c7c3d781dbd, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731259372195 2024-11-10T17:22:57,417 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 785b2965f20c470ebc40a8b7400fac96, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731259373374 2024-11-10T17:22:57,418 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad5494ce8514a10a025a154d8c5ff34, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731259374568 2024-11-10T17:22:57,418 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a56009445334a55b5d214cc04dd38a1, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731259375816 2024-11-10T17:22:57,420 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:57,425 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:57,425 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:22:57,425 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,425 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8400f5fa4706496bb7661d94b371f5f1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d87190153ab94c7792ae15f66a0289c9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8dc5de7c96834eca95fae88f36c4742e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b38333e23eec44f0ae544d76c22684fc] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=46.4 K 2024-11-10T17:22:57,425 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8400f5fa4706496bb7661d94b371f5f1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731259372195 2024-11-10T17:22:57,426 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d87190153ab94c7792ae15f66a0289c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731259373374 2024-11-10T17:22:57,427 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc5de7c96834eca95fae88f36c4742e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731259374568 2024-11-10T17:22:57,427 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b38333e23eec44f0ae544d76c22684fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731259376495 2024-11-10T17:22:57,441 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:57,444 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:57,445 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/08fafee046344c538d79728a99d8d524 is 50, key is test_row_0/B:col10/1731259376495/Put/seqid=0 2024-11-10T17:22:57,449 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110eaf6e579c3e8437b9452f091ff72bc5d_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:57,452 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110eaf6e579c3e8437b9452f091ff72bc5d_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:57,452 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eaf6e579c3e8437b9452f091ff72bc5d_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:57,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742070_1246 (size=13187) 2024-11-10T17:22:57,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/08fafee046344c538d79728a99d8d524 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/08fafee046344c538d79728a99d8d524 2024-11-10T17:22:57,475 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into 08fafee046344c538d79728a99d8d524(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:57,475 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:57,475 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=12, startTime=1731259377414; duration=0sec 2024-11-10T17:22:57,475 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:22:57,475 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:22:57,475 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:22:57,476 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:22:57,476 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:22:57,477 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,477 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a7f4789b8ad949d2a2d2584cce35f12b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a2e38c3cd7de46b38b7792dfce737a69, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/821e4a19b2fd434488d79c2acecceb7c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/deb21a9bb25448da86362b94b3153dfb] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=46.4 K 2024-11-10T17:22:57,477 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f4789b8ad949d2a2d2584cce35f12b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731259372195 2024-11-10T17:22:57,477 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e38c3cd7de46b38b7792dfce737a69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731259373374 2024-11-10T17:22:57,478 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 821e4a19b2fd434488d79c2acecceb7c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731259374568 2024-11-10T17:22:57,478 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting deb21a9bb25448da86362b94b3153dfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731259376495 2024-11-10T17:22:57,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T17:22:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:57,481 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T17:22:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T17:22:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742071_1247 (size=4469) 2024-11-10T17:22:57,495 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:57,495 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/61878d7255a743b78b9a2a9b7fd008ba is 50, key is test_row_0/C:col10/1731259376495/Put/seqid=0 2024-11-10T17:22:57,498 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#204 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:22:57,499 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/8fed5d1ec54a4d06a35f9d4c5be030f1 is 175, key is test_row_0/A:col10/1731259376495/Put/seqid=0 2024-11-10T17:22:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103bfee6c7a4204e0db3679998d752a167_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259376500/Put/seqid=0 2024-11-10T17:22:57,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742073_1249 (size=32141) 2024-11-10T17:22:57,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742074_1250 (size=12454) 2024-11-10T17:22:57,512 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/8fed5d1ec54a4d06a35f9d4c5be030f1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8fed5d1ec54a4d06a35f9d4c5be030f1 2024-11-10T17:22:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:57,520 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103bfee6c7a4204e0db3679998d752a167_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103bfee6c7a4204e0db3679998d752a167_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/58ca371ff318451dadbf3b6b7d8166b3, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/58ca371ff318451dadbf3b6b7d8166b3 is 175, key is test_row_0/A:col10/1731259376500/Put/seqid=0 2024-11-10T17:22:57,525 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into 8fed5d1ec54a4d06a35f9d4c5be030f1(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:57,525 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:57,525 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=12, startTime=1731259377414; duration=0sec 2024-11-10T17:22:57,526 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:57,526 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:22:57,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742072_1248 (size=13187) 2024-11-10T17:22:57,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742075_1251 (size=31255) 2024-11-10T17:22:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. as already flushing 2024-11-10T17:22:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259437698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259437702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259437703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259437811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259437812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259437812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:57,937 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/61878d7255a743b78b9a2a9b7fd008ba as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/61878d7255a743b78b9a2a9b7fd008ba 2024-11-10T17:22:57,943 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into 61878d7255a743b78b9a2a9b7fd008ba(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:22:57,943 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:57,943 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=12, startTime=1731259377414; duration=0sec 2024-11-10T17:22:57,943 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:22:57,943 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:22:57,949 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/58ca371ff318451dadbf3b6b7d8166b3 2024-11-10T17:22:57,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/f794125043904fd7965c7777c60e0556 is 50, key is test_row_0/B:col10/1731259376500/Put/seqid=0 2024-11-10T17:22:57,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742076_1252 (size=12301) 2024-11-10T17:22:58,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T17:22:58,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259438022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259438022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259438022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:58,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51672 deadline: 1731259438331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:58,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:22:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51630 deadline: 1731259438331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51646 deadline: 1731259438331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:22:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:22:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:22:58,370 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/f794125043904fd7965c7777c60e0556 2024-11-10T17:22:58,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/9feb7fa9345049db8e050a6be679ab46 is 50, key is test_row_0/C:col10/1731259376500/Put/seqid=0 2024-11-10T17:22:58,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742077_1253 (size=12301) 2024-11-10T17:22:58,387 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/9feb7fa9345049db8e050a6be679ab46 2024-11-10T17:22:58,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/58ca371ff318451dadbf3b6b7d8166b3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/58ca371ff318451dadbf3b6b7d8166b3 2024-11-10T17:22:58,396 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/58ca371ff318451dadbf3b6b7d8166b3, entries=150, sequenceid=382, filesize=30.5 K 2024-11-10T17:22:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/f794125043904fd7965c7777c60e0556 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f794125043904fd7965c7777c60e0556 2024-11-10T17:22:58,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f794125043904fd7965c7777c60e0556, entries=150, sequenceid=382, filesize=12.0 K 2024-11-10T17:22:58,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/9feb7fa9345049db8e050a6be679ab46 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9feb7fa9345049db8e050a6be679ab46 2024-11-10T17:22:58,409 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9feb7fa9345049db8e050a6be679ab46, entries=150, sequenceid=382, filesize=12.0 K 2024-11-10T17:22:58,410 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for afd93882106705ea245d285e044c07aa in 929ms, sequenceid=382, compaction requested=false 2024-11-10T17:22:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:22:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:22:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T17:22:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T17:22:58,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T17:22:58,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5470 sec 2024-11-10T17:22:58,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5540 sec 2024-11-10T17:22:58,847 INFO [Thread-706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-706. 2024-11-10T17:22:58,848 DEBUG [Thread-706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:58,848 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on afd93882106705ea245d285e044c07aa 2024-11-10T17:22:58,848 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,848 INFO [Thread-712 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-712. 2024-11-10T17:22:58,848 DEBUG [Thread-712 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:58,848 DEBUG [Thread-712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,848 DEBUG [Thread-712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:58,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:22:58,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-10T17:22:58,849 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-10T17:22:58,849 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-10T17:22:58,849 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:58,849 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,849 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:58,849 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,849 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,849 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,849 INFO [Thread-710 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-710. 2024-11-10T17:22:58,849 DEBUG [Thread-710 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:58,849 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:58,849 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:58,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:58,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:58,851 INFO [Thread-708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-708. 2024-11-10T17:22:58,851 DEBUG [Thread-708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:58,851 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,851 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:58,853 INFO [Thread-701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-701. 2024-11-10T17:22:58,853 DEBUG [Thread-701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:22:58,853 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,853 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:22:58,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:22:58,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110434601b9c0974f418eb998ed42dc507d_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259377696/Put/seqid=0 2024-11-10T17:22:58,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742078_1254 (size=12454) 2024-11-10T17:22:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T17:22:59,012 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:22:59,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:22:59,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110434601b9c0974f418eb998ed42dc507d_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110434601b9c0974f418eb998ed42dc507d_afd93882106705ea245d285e044c07aa 2024-11-10T17:22:59,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e4efed40a686455d9ed6564c86e98aec, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:22:59,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e4efed40a686455d9ed6564c86e98aec is 175, key is test_row_0/A:col10/1731259377696/Put/seqid=0 2024-11-10T17:22:59,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742079_1255 (size=31255) 2024-11-10T17:22:59,674 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e4efed40a686455d9ed6564c86e98aec 2024-11-10T17:22:59,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/705903db29fa47e89d29c5f85725d371 is 50, key is test_row_0/B:col10/1731259377696/Put/seqid=0 2024-11-10T17:22:59,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742080_1256 (size=12301) 2024-11-10T17:23:00,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/705903db29fa47e89d29c5f85725d371 2024-11-10T17:23:00,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/04cacd34b4d749f5ba0e5a5d4be65b18 is 50, key is test_row_0/C:col10/1731259377696/Put/seqid=0 2024-11-10T17:23:00,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742081_1257 (size=12301) 2024-11-10T17:23:00,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/04cacd34b4d749f5ba0e5a5d4be65b18 2024-11-10T17:23:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/e4efed40a686455d9ed6564c86e98aec as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e4efed40a686455d9ed6564c86e98aec 2024-11-10T17:23:00,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e4efed40a686455d9ed6564c86e98aec, entries=150, sequenceid=415, filesize=30.5 K 2024-11-10T17:23:00,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/705903db29fa47e89d29c5f85725d371 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/705903db29fa47e89d29c5f85725d371 2024-11-10T17:23:00,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/705903db29fa47e89d29c5f85725d371, entries=150, sequenceid=415, filesize=12.0 K 2024-11-10T17:23:00,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/04cacd34b4d749f5ba0e5a5d4be65b18 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/04cacd34b4d749f5ba0e5a5d4be65b18 2024-11-10T17:23:00,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/04cacd34b4d749f5ba0e5a5d4be65b18, entries=150, sequenceid=415, filesize=12.0 K 2024-11-10T17:23:00,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=6.71 KB/6870 for afd93882106705ea245d285e044c07aa in 1668ms, sequenceid=415, compaction requested=true 2024-11-10T17:23:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:23:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store afd93882106705ea245d285e044c07aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:00,516 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:00,516 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:00,517 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:00,517 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:00,517 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/B is initiating minor compaction (all files) 2024-11-10T17:23:00,517 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/A is initiating minor compaction (all files) 2024-11-10T17:23:00,517 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/A in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:00,517 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/B in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:00,517 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/08fafee046344c538d79728a99d8d524, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f794125043904fd7965c7777c60e0556, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/705903db29fa47e89d29c5f85725d371] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=36.9 K 2024-11-10T17:23:00,517 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8fed5d1ec54a4d06a35f9d4c5be030f1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/58ca371ff318451dadbf3b6b7d8166b3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e4efed40a686455d9ed6564c86e98aec] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=92.4 K 2024-11-10T17:23:00,517 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:00,517 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8fed5d1ec54a4d06a35f9d4c5be030f1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/58ca371ff318451dadbf3b6b7d8166b3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e4efed40a686455d9ed6564c86e98aec] 2024-11-10T17:23:00,518 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fafee046344c538d79728a99d8d524, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731259376495 2024-11-10T17:23:00,518 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fed5d1ec54a4d06a35f9d4c5be030f1, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731259376495 2024-11-10T17:23:00,518 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f794125043904fd7965c7777c60e0556, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731259376500 2024-11-10T17:23:00,518 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58ca371ff318451dadbf3b6b7d8166b3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731259376500 2024-11-10T17:23:00,518 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 705903db29fa47e89d29c5f85725d371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731259377696 2024-11-10T17:23:00,518 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4efed40a686455d9ed6564c86e98aec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731259377696 2024-11-10T17:23:00,528 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#B#compaction#213 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:00,528 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/2420cd21e2254876ba9643a37db88387 is 50, key is test_row_0/B:col10/1731259377696/Put/seqid=0 2024-11-10T17:23:00,529 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:23:00,532 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110533f4f6819e142fbbe3ade6789edc9f0_afd93882106705ea245d285e044c07aa store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:23:00,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742082_1258 (size=13289) 2024-11-10T17:23:00,552 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110533f4f6819e142fbbe3ade6789edc9f0_afd93882106705ea245d285e044c07aa, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:23:00,552 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110533f4f6819e142fbbe3ade6789edc9f0_afd93882106705ea245d285e044c07aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:23:00,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742083_1259 (size=4469) 2024-11-10T17:23:00,941 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/2420cd21e2254876ba9643a37db88387 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/2420cd21e2254876ba9643a37db88387 2024-11-10T17:23:00,945 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/B of afd93882106705ea245d285e044c07aa into 2420cd21e2254876ba9643a37db88387(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:00,945 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:23:00,945 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/B, priority=13, startTime=1731259380516; duration=0sec 2024-11-10T17:23:00,945 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:00,946 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:B 2024-11-10T17:23:00,946 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:00,946 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:00,947 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): afd93882106705ea245d285e044c07aa/C is initiating minor compaction (all files) 2024-11-10T17:23:00,947 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of afd93882106705ea245d285e044c07aa/C in TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:00,947 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/61878d7255a743b78b9a2a9b7fd008ba, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9feb7fa9345049db8e050a6be679ab46, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/04cacd34b4d749f5ba0e5a5d4be65b18] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp, totalSize=36.9 K 2024-11-10T17:23:00,947 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 61878d7255a743b78b9a2a9b7fd008ba, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731259376495 2024-11-10T17:23:00,947 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 9feb7fa9345049db8e050a6be679ab46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731259376500 2024-11-10T17:23:00,948 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 04cacd34b4d749f5ba0e5a5d4be65b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731259377696 2024-11-10T17:23:00,955 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:00,955 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/01cdd65f84d94e4bbf99affe0b7eb8a6 is 50, key is test_row_0/C:col10/1731259377696/Put/seqid=0 2024-11-10T17:23:00,959 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): afd93882106705ea245d285e044c07aa#A#compaction#214 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742084_1260 (size=13289) 2024-11-10T17:23:00,960 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/685e393941bc41fab01b34382d5bac9d is 175, key is test_row_0/A:col10/1731259377696/Put/seqid=0 2024-11-10T17:23:00,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742085_1261 (size=32243) 2024-11-10T17:23:01,364 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/01cdd65f84d94e4bbf99affe0b7eb8a6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/01cdd65f84d94e4bbf99affe0b7eb8a6 2024-11-10T17:23:01,383 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/685e393941bc41fab01b34382d5bac9d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/685e393941bc41fab01b34382d5bac9d 2024-11-10T17:23:01,384 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/C of afd93882106705ea245d285e044c07aa into 01cdd65f84d94e4bbf99affe0b7eb8a6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:01,384 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:23:01,384 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/C, priority=13, startTime=1731259380516; duration=0sec 2024-11-10T17:23:01,385 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:01,385 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:C 2024-11-10T17:23:01,387 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in afd93882106705ea245d285e044c07aa/A of afd93882106705ea245d285e044c07aa into 685e393941bc41fab01b34382d5bac9d(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:01,387 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for afd93882106705ea245d285e044c07aa: 2024-11-10T17:23:01,387 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., storeName=afd93882106705ea245d285e044c07aa/A, priority=13, startTime=1731259380516; duration=0sec 2024-11-10T17:23:01,387 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:01,387 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: afd93882106705ea245d285e044c07aa:A 2024-11-10T17:23:02,253 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-10T17:23:02,253 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:02,253 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:02,253 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:02,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:02,313 INFO [Thread-703 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-703. 2024-11-10T17:23:02,313 DEBUG [Thread-703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:02,313 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:02,313 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:02,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3483 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3169 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1452 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4356 rows 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1435 2024-11-10T17:23:02,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4305 rows 2024-11-10T17:23:02,314 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:23:02,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:23:02,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:02,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:02,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:02,315 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T17:23:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T17:23:02,318 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259382317"}]},"ts":"1731259382317"} 2024-11-10T17:23:02,319 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T17:23:02,319 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T17:23:02,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:23:02,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, UNASSIGN}] 2024-11-10T17:23:02,322 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, UNASSIGN 2024-11-10T17:23:02,323 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:02,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, UNASSIGN because future has completed 2024-11-10T17:23:02,324 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:23:02,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:23:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T17:23:02,477 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close afd93882106705ea245d285e044c07aa 2024-11-10T17:23:02,477 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:23:02,477 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing afd93882106705ea245d285e044c07aa, disabling compactions & flushes 2024-11-10T17:23:02,477 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:02,477 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:02,477 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. after waiting 0 ms 2024-11-10T17:23:02,477 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:02,477 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing afd93882106705ea245d285e044c07aa 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-10T17:23:02,483 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a7953b4507b41cbabb3dd53fff67854_afd93882106705ea245d285e044c07aa is 50, key is test_row_0/A:col10/1731259382252/Put/seqid=0 2024-11-10T17:23:02,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742086_1262 (size=12454) 2024-11-10T17:23:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T17:23:02,887 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:02,891 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a7953b4507b41cbabb3dd53fff67854_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a7953b4507b41cbabb3dd53fff67854_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:02,892 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6055273b96084018ae32860133339a68, store: [table=TestAcidGuarantees family=A region=afd93882106705ea245d285e044c07aa] 2024-11-10T17:23:02,892 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6055273b96084018ae32860133339a68 is 175, key is test_row_0/A:col10/1731259382252/Put/seqid=0 2024-11-10T17:23:02,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742087_1263 (size=31255) 2024-11-10T17:23:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T17:23:03,296 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6055273b96084018ae32860133339a68 2024-11-10T17:23:03,303 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/88ff61d1f018497195ac115f67c7c761 is 50, key is test_row_0/B:col10/1731259382252/Put/seqid=0 2024-11-10T17:23:03,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742088_1264 (size=12301) 2024-11-10T17:23:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T17:23:03,707 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/88ff61d1f018497195ac115f67c7c761 2024-11-10T17:23:03,714 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/d4f24e933026461f8f5d3311aac012ba is 50, key is test_row_0/C:col10/1731259382252/Put/seqid=0 2024-11-10T17:23:03,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742089_1265 (size=12301) 2024-11-10T17:23:04,118 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/d4f24e933026461f8f5d3311aac012ba 2024-11-10T17:23:04,124 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/A/6055273b96084018ae32860133339a68 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6055273b96084018ae32860133339a68 2024-11-10T17:23:04,130 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6055273b96084018ae32860133339a68, entries=150, sequenceid=424, filesize=30.5 K 2024-11-10T17:23:04,131 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/B/88ff61d1f018497195ac115f67c7c761 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/88ff61d1f018497195ac115f67c7c761 2024-11-10T17:23:04,136 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/88ff61d1f018497195ac115f67c7c761, entries=150, sequenceid=424, filesize=12.0 K 2024-11-10T17:23:04,137 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/.tmp/C/d4f24e933026461f8f5d3311aac012ba as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d4f24e933026461f8f5d3311aac012ba 2024-11-10T17:23:04,141 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d4f24e933026461f8f5d3311aac012ba, entries=150, sequenceid=424, filesize=12.0 K 2024-11-10T17:23:04,142 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for afd93882106705ea245d285e044c07aa in 1664ms, sequenceid=424, compaction requested=false 2024-11-10T17:23:04,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/faef8c01864748449f242a9cacab4bcd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/f4857245ef344549b611b27b92890e89, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d396149be8c4b2cac497b1b4e118ce3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aaa987f67d464959a459d31b0b8e4b58, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/b3484b8a054f4e2292b559b85a5864b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/fd2f1bcc2ee74252a06d2a9ebf196d13, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/27dab2c7796846d1b9ff09b6251cc992, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/c30da92e8a494d9099009bea9a867086, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6fd601a64e104c468f3fac6ede984404, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6859906bbcb845e29a8db3e1f43e64f5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2701929317184c4dae976ee1045ea9f8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4e4f30b0567f4f2a9ce105de343538b2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e0e820b92e5e4c37b614bfa720a00aaa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/dee66fc6552244c4939f8705d67b0aae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/7b942e9d621d49a580d484476486609d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9651cf9a836943ddb2361a8543aa3fe5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/d6dcdce09545468db98ba646b6af6dd6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d3d56af65da48409eb97730a1e36478, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4848c466b9fd4e36b345a1b03537062f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2a38e3959f764a9e88124470746bfe65, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/ec7b7863f7cb47b093fa41e24e60b11f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9f2a55754b604904af5c1c7c3d781dbd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/3eb228aa6d3944c98b7be369f5e02ad5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/785b2965f20c470ebc40a8b7400fac96, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aad5494ce8514a10a025a154d8c5ff34, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8a56009445334a55b5d214cc04dd38a1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8fed5d1ec54a4d06a35f9d4c5be030f1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/58ca371ff318451dadbf3b6b7d8166b3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e4efed40a686455d9ed6564c86e98aec] to archive 2024-11-10T17:23:04,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:23:04,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/faef8c01864748449f242a9cacab4bcd to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/faef8c01864748449f242a9cacab4bcd 2024-11-10T17:23:04,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/f4857245ef344549b611b27b92890e89 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/f4857245ef344549b611b27b92890e89 2024-11-10T17:23:04,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d396149be8c4b2cac497b1b4e118ce3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d396149be8c4b2cac497b1b4e118ce3 2024-11-10T17:23:04,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aaa987f67d464959a459d31b0b8e4b58 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aaa987f67d464959a459d31b0b8e4b58 2024-11-10T17:23:04,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/b3484b8a054f4e2292b559b85a5864b6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/b3484b8a054f4e2292b559b85a5864b6 2024-11-10T17:23:04,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/fd2f1bcc2ee74252a06d2a9ebf196d13 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/fd2f1bcc2ee74252a06d2a9ebf196d13 2024-11-10T17:23:04,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/27dab2c7796846d1b9ff09b6251cc992 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/27dab2c7796846d1b9ff09b6251cc992 2024-11-10T17:23:04,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/c30da92e8a494d9099009bea9a867086 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/c30da92e8a494d9099009bea9a867086 2024-11-10T17:23:04,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6fd601a64e104c468f3fac6ede984404 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6fd601a64e104c468f3fac6ede984404 2024-11-10T17:23:04,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6859906bbcb845e29a8db3e1f43e64f5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6859906bbcb845e29a8db3e1f43e64f5 2024-11-10T17:23:04,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2701929317184c4dae976ee1045ea9f8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2701929317184c4dae976ee1045ea9f8 2024-11-10T17:23:04,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4e4f30b0567f4f2a9ce105de343538b2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4e4f30b0567f4f2a9ce105de343538b2 2024-11-10T17:23:04,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e0e820b92e5e4c37b614bfa720a00aaa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e0e820b92e5e4c37b614bfa720a00aaa 2024-11-10T17:23:04,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/dee66fc6552244c4939f8705d67b0aae to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/dee66fc6552244c4939f8705d67b0aae 2024-11-10T17:23:04,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/7b942e9d621d49a580d484476486609d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/7b942e9d621d49a580d484476486609d 2024-11-10T17:23:04,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9651cf9a836943ddb2361a8543aa3fe5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9651cf9a836943ddb2361a8543aa3fe5 2024-11-10T17:23:04,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/d6dcdce09545468db98ba646b6af6dd6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/d6dcdce09545468db98ba646b6af6dd6 2024-11-10T17:23:04,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d3d56af65da48409eb97730a1e36478 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4d3d56af65da48409eb97730a1e36478 2024-11-10T17:23:04,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4848c466b9fd4e36b345a1b03537062f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/4848c466b9fd4e36b345a1b03537062f 2024-11-10T17:23:04,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2a38e3959f764a9e88124470746bfe65 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/2a38e3959f764a9e88124470746bfe65 2024-11-10T17:23:04,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/ec7b7863f7cb47b093fa41e24e60b11f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/ec7b7863f7cb47b093fa41e24e60b11f 2024-11-10T17:23:04,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9f2a55754b604904af5c1c7c3d781dbd to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/9f2a55754b604904af5c1c7c3d781dbd 2024-11-10T17:23:04,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/3eb228aa6d3944c98b7be369f5e02ad5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/3eb228aa6d3944c98b7be369f5e02ad5 2024-11-10T17:23:04,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/785b2965f20c470ebc40a8b7400fac96 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/785b2965f20c470ebc40a8b7400fac96 2024-11-10T17:23:04,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aad5494ce8514a10a025a154d8c5ff34 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/aad5494ce8514a10a025a154d8c5ff34 2024-11-10T17:23:04,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8a56009445334a55b5d214cc04dd38a1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8a56009445334a55b5d214cc04dd38a1 2024-11-10T17:23:04,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8fed5d1ec54a4d06a35f9d4c5be030f1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/8fed5d1ec54a4d06a35f9d4c5be030f1 2024-11-10T17:23:04,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/58ca371ff318451dadbf3b6b7d8166b3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/58ca371ff318451dadbf3b6b7d8166b3 2024-11-10T17:23:04,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e4efed40a686455d9ed6564c86e98aec to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/e4efed40a686455d9ed6564c86e98aec 2024-11-10T17:23:04,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/cbf529ddb4d942eea0cd43a68b75cfe0, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/3a0fa5e1dff245b1b0f484270fb672cb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c690d882394e44cfa72ea5c1775ef439, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ceafe0e310054ec19c784dfd859f5ed1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ff12dd22f5344f649c76ec5dc8e6a652, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d291bf7ec7a747bdb373b9f0dcb81882, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d5235d9d08744a72b15d825816f0cfd8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/401564ca14944b5a91eae778125b8681, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b39f7d0c86394be9813297fc3b1bf6e9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/6e45c560b4714955ad031d27f723397c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/9f53d2ebeab84f10aecaaf86135dd191, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/84b33755223d4bd6be129f7561662be3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/58dac864dd42475cbb52371e1a25ef8b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8a8f7371b95448e28c586c2e13523510, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b4a8ee13c4594681a3f84cc9e9de4063, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f0f2955f554f4fef9979c6fbcbd88cac, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c9a71878d3fe487eb37c4be00595c116, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8b82008f2cd0402c875e6b5c247f5ce3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/60ff5e0c74124a1685ba1e8f6af45e9b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8181ef62fbbd4a4fb6b5e2391aac922a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/831fc4b9a8b84e0982008f58169a762d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8400f5fa4706496bb7661d94b371f5f1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ec3ac21504e74e37841039acab38ec88, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d87190153ab94c7792ae15f66a0289c9, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8dc5de7c96834eca95fae88f36c4742e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/08fafee046344c538d79728a99d8d524, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b38333e23eec44f0ae544d76c22684fc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f794125043904fd7965c7777c60e0556, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/705903db29fa47e89d29c5f85725d371] to archive 2024-11-10T17:23:04,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:23:04,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/cbf529ddb4d942eea0cd43a68b75cfe0 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/cbf529ddb4d942eea0cd43a68b75cfe0 2024-11-10T17:23:04,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/3a0fa5e1dff245b1b0f484270fb672cb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/3a0fa5e1dff245b1b0f484270fb672cb 2024-11-10T17:23:04,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c690d882394e44cfa72ea5c1775ef439 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c690d882394e44cfa72ea5c1775ef439 2024-11-10T17:23:04,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ceafe0e310054ec19c784dfd859f5ed1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ceafe0e310054ec19c784dfd859f5ed1 2024-11-10T17:23:04,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ff12dd22f5344f649c76ec5dc8e6a652 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ff12dd22f5344f649c76ec5dc8e6a652 2024-11-10T17:23:04,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d291bf7ec7a747bdb373b9f0dcb81882 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d291bf7ec7a747bdb373b9f0dcb81882 2024-11-10T17:23:04,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d5235d9d08744a72b15d825816f0cfd8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d5235d9d08744a72b15d825816f0cfd8 2024-11-10T17:23:04,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/401564ca14944b5a91eae778125b8681 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/401564ca14944b5a91eae778125b8681 2024-11-10T17:23:04,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b39f7d0c86394be9813297fc3b1bf6e9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b39f7d0c86394be9813297fc3b1bf6e9 2024-11-10T17:23:04,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/6e45c560b4714955ad031d27f723397c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/6e45c560b4714955ad031d27f723397c 2024-11-10T17:23:04,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/9f53d2ebeab84f10aecaaf86135dd191 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/9f53d2ebeab84f10aecaaf86135dd191 2024-11-10T17:23:04,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/84b33755223d4bd6be129f7561662be3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/84b33755223d4bd6be129f7561662be3 2024-11-10T17:23:04,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/58dac864dd42475cbb52371e1a25ef8b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/58dac864dd42475cbb52371e1a25ef8b 2024-11-10T17:23:04,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8a8f7371b95448e28c586c2e13523510 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8a8f7371b95448e28c586c2e13523510 2024-11-10T17:23:04,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b4a8ee13c4594681a3f84cc9e9de4063 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b4a8ee13c4594681a3f84cc9e9de4063 2024-11-10T17:23:04,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f0f2955f554f4fef9979c6fbcbd88cac to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f0f2955f554f4fef9979c6fbcbd88cac 2024-11-10T17:23:04,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c9a71878d3fe487eb37c4be00595c116 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/c9a71878d3fe487eb37c4be00595c116 2024-11-10T17:23:04,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8b82008f2cd0402c875e6b5c247f5ce3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8b82008f2cd0402c875e6b5c247f5ce3 2024-11-10T17:23:04,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/60ff5e0c74124a1685ba1e8f6af45e9b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/60ff5e0c74124a1685ba1e8f6af45e9b 2024-11-10T17:23:04,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8181ef62fbbd4a4fb6b5e2391aac922a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8181ef62fbbd4a4fb6b5e2391aac922a 2024-11-10T17:23:04,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/831fc4b9a8b84e0982008f58169a762d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/831fc4b9a8b84e0982008f58169a762d 2024-11-10T17:23:04,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8400f5fa4706496bb7661d94b371f5f1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8400f5fa4706496bb7661d94b371f5f1 2024-11-10T17:23:04,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ec3ac21504e74e37841039acab38ec88 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/ec3ac21504e74e37841039acab38ec88 2024-11-10T17:23:04,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d87190153ab94c7792ae15f66a0289c9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/d87190153ab94c7792ae15f66a0289c9 2024-11-10T17:23:04,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8dc5de7c96834eca95fae88f36c4742e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/8dc5de7c96834eca95fae88f36c4742e 2024-11-10T17:23:04,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/08fafee046344c538d79728a99d8d524 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/08fafee046344c538d79728a99d8d524 2024-11-10T17:23:04,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b38333e23eec44f0ae544d76c22684fc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/b38333e23eec44f0ae544d76c22684fc 2024-11-10T17:23:04,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f794125043904fd7965c7777c60e0556 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/f794125043904fd7965c7777c60e0556 2024-11-10T17:23:04,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/705903db29fa47e89d29c5f85725d371 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/705903db29fa47e89d29c5f85725d371 2024-11-10T17:23:04,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b6702d4a3c484dcba75a314998cad4aa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46dfff49765542a394aff8b8649efa76, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/cd04162849f34fa1a72e910f8cd6c4ed, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/fbbd3a493e0942daaba7ba7f15ccb9cd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46be85c77e984dc68d8e04be7136fd0e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/e869737f21ff443cacca5e689604819d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4d07f414a3554dc4957ee41e908ebd2b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/54e22f84c2fd43a0988d21b2fa4fc8a6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/90548315520e4b229f45b8dbcfc30f21, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/02f2bde3143d447cbb39241ad0454ecc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/84d92304e40645999df35b8ed33cb4a1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/dc884d7e32664b68a275aecc4a00db68, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/edf8481ffe614cc38929a8377fecb8b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/daef474b7b274646b2af332079adbb44, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/c3ff9181c87d49a9a0a28d934414890f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b5833fbc89c749cea2d5a1aa0265847e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9a9b420d6191417da43ef7ce408f3e22, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4155722b8dc04da28e1caa7171a86eca, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/6af3d06de601411e9733ee81250a1f64, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/665e4afddd2146ee83ede4b39e948d03, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/513e817797b649e1856a4eeaddb4b745, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a7f4789b8ad949d2a2d2584cce35f12b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d72bcdcf482c4cbbb9330ef37b0d9402, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a2e38c3cd7de46b38b7792dfce737a69, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/821e4a19b2fd434488d79c2acecceb7c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/61878d7255a743b78b9a2a9b7fd008ba, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/deb21a9bb25448da86362b94b3153dfb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9feb7fa9345049db8e050a6be679ab46, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/04cacd34b4d749f5ba0e5a5d4be65b18] to archive 2024-11-10T17:23:04,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:23:04,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b6702d4a3c484dcba75a314998cad4aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b6702d4a3c484dcba75a314998cad4aa 2024-11-10T17:23:04,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46dfff49765542a394aff8b8649efa76 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46dfff49765542a394aff8b8649efa76 2024-11-10T17:23:04,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/cd04162849f34fa1a72e910f8cd6c4ed to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/cd04162849f34fa1a72e910f8cd6c4ed 2024-11-10T17:23:04,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/fbbd3a493e0942daaba7ba7f15ccb9cd to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/fbbd3a493e0942daaba7ba7f15ccb9cd 2024-11-10T17:23:04,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46be85c77e984dc68d8e04be7136fd0e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/46be85c77e984dc68d8e04be7136fd0e 2024-11-10T17:23:04,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/e869737f21ff443cacca5e689604819d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/e869737f21ff443cacca5e689604819d 2024-11-10T17:23:04,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4d07f414a3554dc4957ee41e908ebd2b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4d07f414a3554dc4957ee41e908ebd2b 2024-11-10T17:23:04,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/54e22f84c2fd43a0988d21b2fa4fc8a6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/54e22f84c2fd43a0988d21b2fa4fc8a6 2024-11-10T17:23:04,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/90548315520e4b229f45b8dbcfc30f21 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/90548315520e4b229f45b8dbcfc30f21 2024-11-10T17:23:04,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/02f2bde3143d447cbb39241ad0454ecc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/02f2bde3143d447cbb39241ad0454ecc 2024-11-10T17:23:04,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/84d92304e40645999df35b8ed33cb4a1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/84d92304e40645999df35b8ed33cb4a1 2024-11-10T17:23:04,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/dc884d7e32664b68a275aecc4a00db68 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/dc884d7e32664b68a275aecc4a00db68 2024-11-10T17:23:04,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/edf8481ffe614cc38929a8377fecb8b6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/edf8481ffe614cc38929a8377fecb8b6 2024-11-10T17:23:04,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/daef474b7b274646b2af332079adbb44 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/daef474b7b274646b2af332079adbb44 2024-11-10T17:23:04,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/c3ff9181c87d49a9a0a28d934414890f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/c3ff9181c87d49a9a0a28d934414890f 2024-11-10T17:23:04,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b5833fbc89c749cea2d5a1aa0265847e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/b5833fbc89c749cea2d5a1aa0265847e 2024-11-10T17:23:04,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9a9b420d6191417da43ef7ce408f3e22 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9a9b420d6191417da43ef7ce408f3e22 2024-11-10T17:23:04,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4155722b8dc04da28e1caa7171a86eca to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/4155722b8dc04da28e1caa7171a86eca 2024-11-10T17:23:04,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/6af3d06de601411e9733ee81250a1f64 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/6af3d06de601411e9733ee81250a1f64 2024-11-10T17:23:04,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/665e4afddd2146ee83ede4b39e948d03 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/665e4afddd2146ee83ede4b39e948d03 2024-11-10T17:23:04,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/513e817797b649e1856a4eeaddb4b745 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/513e817797b649e1856a4eeaddb4b745 2024-11-10T17:23:04,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a7f4789b8ad949d2a2d2584cce35f12b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a7f4789b8ad949d2a2d2584cce35f12b 2024-11-10T17:23:04,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d72bcdcf482c4cbbb9330ef37b0d9402 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d72bcdcf482c4cbbb9330ef37b0d9402 2024-11-10T17:23:04,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a2e38c3cd7de46b38b7792dfce737a69 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/a2e38c3cd7de46b38b7792dfce737a69 2024-11-10T17:23:04,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/821e4a19b2fd434488d79c2acecceb7c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/821e4a19b2fd434488d79c2acecceb7c 2024-11-10T17:23:04,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/61878d7255a743b78b9a2a9b7fd008ba to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/61878d7255a743b78b9a2a9b7fd008ba 2024-11-10T17:23:04,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/deb21a9bb25448da86362b94b3153dfb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/deb21a9bb25448da86362b94b3153dfb 2024-11-10T17:23:04,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9feb7fa9345049db8e050a6be679ab46 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/9feb7fa9345049db8e050a6be679ab46 2024-11-10T17:23:04,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/04cacd34b4d749f5ba0e5a5d4be65b18 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/04cacd34b4d749f5ba0e5a5d4be65b18 2024-11-10T17:23:04,251 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/recovered.edits/427.seqid, newMaxSeqId=427, maxSeqId=4 2024-11-10T17:23:04,253 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. 2024-11-10T17:23:04,253 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for afd93882106705ea245d285e044c07aa: Waiting for close lock at 1731259382477Running coprocessor pre-close hooks at 1731259382477Disabling compacts and flushes for region at 1731259382477Disabling writes for close at 1731259382477Obtaining lock to block concurrent updates at 1731259382477Preparing flush snapshotting stores in afd93882106705ea245d285e044c07aa at 1731259382477Finished memstore snapshotting TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731259382478 (+1 ms)Flushing stores of TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa. at 1731259382478Flushing afd93882106705ea245d285e044c07aa/A: creating writer at 1731259382478Flushing mob file afd93882106705ea245d285e044c07aa/A: appending metadata at 1731259382483 (+5 ms)Flushing mob file afd93882106705ea245d285e044c07aa/A: closing flushed file at 1731259382483Flushing afd93882106705ea245d285e044c07aa/A: appending metadata at 1731259382892 (+409 ms)Flushing afd93882106705ea245d285e044c07aa/A: closing flushed file at 1731259382892Flushing afd93882106705ea245d285e044c07aa/B: creating writer at 1731259383300 (+408 ms)Flushing afd93882106705ea245d285e044c07aa/B: appending metadata at 1731259383302 (+2 ms)Flushing afd93882106705ea245d285e044c07aa/B: closing flushed file at 1731259383302Flushing afd93882106705ea245d285e044c07aa/C: creating writer at 1731259383711 (+409 ms)Flushing afd93882106705ea245d285e044c07aa/C: appending metadata at 1731259383714 (+3 ms)Flushing afd93882106705ea245d285e044c07aa/C: closing flushed file at 1731259383714Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52a314cc: reopening flushed file at 1731259384123 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e71ae1a: reopening flushed file at 1731259384130 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dd36445: reopening flushed file at 1731259384136 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for afd93882106705ea245d285e044c07aa in 1664ms, sequenceid=424, compaction requested=false at 1731259384142 (+6 ms)Writing region close event to WAL at 1731259384248 (+106 ms)Running coprocessor post-close hooks at 1731259384252 (+4 ms)Closed at 1731259384253 (+1 ms) 2024-11-10T17:23:04,255 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,255 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=afd93882106705ea245d285e044c07aa, regionState=CLOSED 2024-11-10T17:23:04,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:23:04,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T17:23:04,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure afd93882106705ea245d285e044c07aa, server=8e3de9160f9f,36421,1731259326553 in 1.9330 sec 2024-11-10T17:23:04,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-10T17:23:04,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=afd93882106705ea245d285e044c07aa, UNASSIGN in 1.9370 sec 2024-11-10T17:23:04,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T17:23:04,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-10T17:23:04,263 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259384263"}]},"ts":"1731259384263"} 2024-11-10T17:23:04,265 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T17:23:04,265 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T17:23:04,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9500 sec 2024-11-10T17:23:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T17:23:04,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:04,462 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T17:23:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:04,464 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T17:23:04,465 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:04,468 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,470 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/recovered.edits] 2024-11-10T17:23:04,473 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6055273b96084018ae32860133339a68 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/6055273b96084018ae32860133339a68 2024-11-10T17:23:04,475 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/685e393941bc41fab01b34382d5bac9d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/A/685e393941bc41fab01b34382d5bac9d 2024-11-10T17:23:04,478 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/2420cd21e2254876ba9643a37db88387 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/2420cd21e2254876ba9643a37db88387 2024-11-10T17:23:04,479 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/88ff61d1f018497195ac115f67c7c761 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/B/88ff61d1f018497195ac115f67c7c761 2024-11-10T17:23:04,482 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/01cdd65f84d94e4bbf99affe0b7eb8a6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/01cdd65f84d94e4bbf99affe0b7eb8a6 2024-11-10T17:23:04,483 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d4f24e933026461f8f5d3311aac012ba to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/C/d4f24e933026461f8f5d3311aac012ba 2024-11-10T17:23:04,486 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/recovered.edits/427.seqid to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa/recovered.edits/427.seqid 2024-11-10T17:23:04,487 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,487 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T17:23:04,487 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T17:23:04,488 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T17:23:04,492 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110073961ef7c134087ae6607cd8249623b_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110073961ef7c134087ae6607cd8249623b_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,494 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100d4710eba9fb4fa1b7eb24de3170a85b_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100d4710eba9fb4fa1b7eb24de3170a85b_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,495 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100f6cf6b36a904dd4aab187b34ffcf830_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100f6cf6b36a904dd4aab187b34ffcf830_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,496 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102726a1558fca4aec87cc43e59ff663cc_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102726a1558fca4aec87cc43e59ff663cc_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,497 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103bfee6c7a4204e0db3679998d752a167_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103bfee6c7a4204e0db3679998d752a167_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,498 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110434601b9c0974f418eb998ed42dc507d_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110434601b9c0974f418eb998ed42dc507d_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,499 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e2550cd046343a7a9b03dc6144f1b78_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e2550cd046343a7a9b03dc6144f1b78_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,501 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105c801e6c6d3345fdaef4e4673670b22d_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105c801e6c6d3345fdaef4e4673670b22d_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,502 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106185b9817aa74c73a87bd3461e6b1cc5_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106185b9817aa74c73a87bd3461e6b1cc5_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,503 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061f7c39bf7004a2eb3095449ff03b652_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111061f7c39bf7004a2eb3095449ff03b652_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a7953b4507b41cbabb3dd53fff67854_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a7953b4507b41cbabb3dd53fff67854_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,506 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106c9f3e991d1c42449fd2b9cc611a20d6_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106c9f3e991d1c42449fd2b9cc611a20d6_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107c9b9d7cf81643869ccfdcc850b8b067_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107c9b9d7cf81643869ccfdcc850b8b067_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,508 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d2d0b33bbf94c04bd3340e5d6d8306d_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d2d0b33bbf94c04bd3340e5d6d8306d_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,509 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fb12bfad5fb4fb2b1482cb1695d642c_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fb12bfad5fb4fb2b1482cb1695d642c_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,510 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108271636dff014208936d1c7879cea027_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108271636dff014208936d1c7879cea027_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a35f3be9ca0747d2ad896004078610d9_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a35f3be9ca0747d2ad896004078610d9_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,513 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b45400a3036f4732b9271a25aaacbfde_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b45400a3036f4732b9271a25aaacbfde_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,514 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be553263aebd4d788462747d4dae0468_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be553263aebd4d788462747d4dae0468_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,515 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bea292926de6462da70ba2829229a1c0_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bea292926de6462da70ba2829229a1c0_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,516 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ec989e5550de4c94bfa3eca693b8ad87_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ec989e5550de4c94bfa3eca693b8ad87_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,517 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f9b0c467baa442839b62be2f9c5c14d2_afd93882106705ea245d285e044c07aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f9b0c467baa442839b62be2f9c5c14d2_afd93882106705ea245d285e044c07aa 2024-11-10T17:23:04,518 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T17:23:04,520 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:04,523 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T17:23:04,525 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T17:23:04,526 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:04,526 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T17:23:04,526 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731259384526"}]},"ts":"9223372036854775807"} 2024-11-10T17:23:04,528 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T17:23:04,528 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => afd93882106705ea245d285e044c07aa, NAME => 'TestAcidGuarantees,,1731259355602.afd93882106705ea245d285e044c07aa.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T17:23:04,528 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T17:23:04,528 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731259384528"}]},"ts":"9223372036854775807"} 2024-11-10T17:23:04,530 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T17:23:04,531 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:04,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-10T17:23:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T17:23:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T17:23:04,572 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:04,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=245 (was 239) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-255130881_22 at /127.0.0.1:39678 [Waiting for operation #1417] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-255130881_22 at /127.0.0.1:56666 [Waiting for operation #933] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2144637464_22 at /127.0.0.1:38568 [Waiting for operation #1016] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2144637464_22 at /127.0.0.1:56662 [Waiting for operation #945] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=485 (was 456) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6200 (was 6335) 2024-11-10T17:23:04,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=245, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=485, ProcessCount=11, AvailableMemoryMB=6199 2024-11-10T17:23:04,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:23:04,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:23:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:04,594 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T17:23:04,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-10T17:23:04,594 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-10T17:23:04,595 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T17:23:04,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742090_1266 (size=959) 2024-11-10T17:23:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-10T17:23:04,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T17:23:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-10T17:23:05,003 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:23:05,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742091_1267 (size=53) 2024-11-10T17:23:05,008 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:23:05,008 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 84febc6fa98bac2754e6c1b07a211911, disabling compactions & flushes 2024-11-10T17:23:05,008 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,008 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,008 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. after waiting 0 ms 2024-11-10T17:23:05,008 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,008 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,008 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 84febc6fa98bac2754e6c1b07a211911: Waiting for close lock at 1731259385008Disabling compacts and flushes for region at 1731259385008Disabling writes for close at 1731259385008Writing region close event to WAL at 1731259385008Closed at 1731259385008 2024-11-10T17:23:05,009 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T17:23:05,009 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731259385009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731259385009"}]},"ts":"1731259385009"} 2024-11-10T17:23:05,011 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T17:23:05,011 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T17:23:05,012 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259385011"}]},"ts":"1731259385011"} 2024-11-10T17:23:05,013 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T17:23:05,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, ASSIGN}] 2024-11-10T17:23:05,014 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, ASSIGN 2024-11-10T17:23:05,014 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, ASSIGN; state=OFFLINE, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=false 2024-11-10T17:23:05,165 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=84febc6fa98bac2754e6c1b07a211911, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, ASSIGN because future has completed 2024-11-10T17:23:05,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:23:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-10T17:23:05,321 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,321 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:23:05,321 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,321 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:23:05,322 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,322 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,323 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,324 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84febc6fa98bac2754e6c1b07a211911 columnFamilyName A 2024-11-10T17:23:05,324 DEBUG [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:05,324 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] regionserver.HStore(327): Store=84febc6fa98bac2754e6c1b07a211911/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:05,325 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,326 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84febc6fa98bac2754e6c1b07a211911 columnFamilyName B 2024-11-10T17:23:05,326 DEBUG [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:05,326 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] regionserver.HStore(327): Store=84febc6fa98bac2754e6c1b07a211911/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:05,326 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,327 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84febc6fa98bac2754e6c1b07a211911 columnFamilyName C 2024-11-10T17:23:05,327 DEBUG [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:05,327 INFO [StoreOpener-84febc6fa98bac2754e6c1b07a211911-1 {}] regionserver.HStore(327): Store=84febc6fa98bac2754e6c1b07a211911/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:05,327 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,328 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,328 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,329 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,329 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,330 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:23:05,330 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,332 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:23:05,333 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 84febc6fa98bac2754e6c1b07a211911; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66602616, jitterRate=-0.007543683052062988}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:23:05,333 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,333 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 84febc6fa98bac2754e6c1b07a211911: Running coprocessor pre-open hook at 1731259385322Writing region info on filesystem at 1731259385322Initializing all the Stores at 1731259385322Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259385322Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259385323 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259385323Cleaning up temporary data from old regions at 1731259385329 (+6 ms)Running coprocessor post-open hooks at 1731259385333 (+4 ms)Region opened successfully at 1731259385333 2024-11-10T17:23:05,334 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., pid=57, masterSystemTime=1731259385318 2024-11-10T17:23:05,335 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,335 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=84febc6fa98bac2754e6c1b07a211911, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:23:05,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T17:23:05,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 in 172 msec 2024-11-10T17:23:05,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-10T17:23:05,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, ASSIGN in 327 msec 2024-11-10T17:23:05,342 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T17:23:05,342 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259385342"}]},"ts":"1731259385342"} 2024-11-10T17:23:05,343 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T17:23:05,344 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T17:23:05,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 752 msec 2024-11-10T17:23:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-10T17:23:05,732 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:05,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,735 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@628cd56d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f612820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,737 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T17:23:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,40007,1731259325831 2024-11-10T17:23:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@34854d9a 2024-11-10T17:23:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T17:23:05,741 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T17:23:05,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,744 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db4321e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,745 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9c4e98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,747 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267929a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,748 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7684a95f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,750 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63cedc07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,751 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e09d2ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7051f15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451fbb59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,755 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f04a94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffd81bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,758 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e93f7a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46559b15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e1a7657, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f65dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,763 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc69579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,764 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c1bba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,766 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21df741e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383917c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:05,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:05,768 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@501f5bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:05,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61509bad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:05,770 DEBUG [Thread-1239 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,773 DEBUG [Thread-1241 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,773 DEBUG [Thread-1226 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-10T17:23:05,775 DEBUG [Thread-1228 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,775 DEBUG [Thread-1237 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,775 DEBUG [Thread-1245 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,776 DEBUG [Thread-1228 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T17:23:05,776 DEBUG [Thread-1228 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,777 DEBUG [Thread-1230 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,778 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,778 DEBUG [Thread-1232 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,779 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:05,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,780 DEBUG [Thread-1234 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,789 DEBUG [Thread-1243 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:05,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:23:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:05,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:23:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:05,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/c7f59cf4e5b645269991120d631b529a is 50, key is test_row_0/A:col10/1731259385788/Put/seqid=0 2024-11-10T17:23:05,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259445847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259445850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259445851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259445853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259445853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742092_1268 (size=12001) 2024-11-10T17:23:05,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/c7f59cf4e5b645269991120d631b529a 2024-11-10T17:23:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T17:23:05,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/aa20ada6338749e1b2e9dd3f25e6d45f is 50, key is test_row_0/B:col10/1731259385788/Put/seqid=0 2024-11-10T17:23:05,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:05,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:05,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:05,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742093_1269 (size=12001) 2024-11-10T17:23:05,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259445962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259445971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259445971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259445971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259445972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T17:23:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259446171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259446182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259446182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259446182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259446182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/aa20ada6338749e1b2e9dd3f25e6d45f 2024-11-10T17:23:06,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/478cfb48fb1e44f8a312a2e75088f96b is 50, key is test_row_0/C:col10/1731259385788/Put/seqid=0 2024-11-10T17:23:06,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742094_1270 (size=12001) 2024-11-10T17:23:06,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/478cfb48fb1e44f8a312a2e75088f96b 2024-11-10T17:23:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T17:23:06,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/c7f59cf4e5b645269991120d631b529a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c7f59cf4e5b645269991120d631b529a 2024-11-10T17:23:06,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c7f59cf4e5b645269991120d631b529a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T17:23:06,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/aa20ada6338749e1b2e9dd3f25e6d45f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/aa20ada6338749e1b2e9dd3f25e6d45f 2024-11-10T17:23:06,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/aa20ada6338749e1b2e9dd3f25e6d45f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T17:23:06,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/478cfb48fb1e44f8a312a2e75088f96b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/478cfb48fb1e44f8a312a2e75088f96b 2024-11-10T17:23:06,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/478cfb48fb1e44f8a312a2e75088f96b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T17:23:06,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 84febc6fa98bac2754e6c1b07a211911 in 645ms, sequenceid=14, compaction requested=false 2024-11-10T17:23:06,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:06,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:23:06,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/801c72ebf2a44cd09fad84f91cbc6cb2 is 50, key is test_row_0/A:col10/1731259386482/Put/seqid=0 2024-11-10T17:23:06,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259446495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259446496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259446497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259446498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259446499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742095_1271 (size=14341) 2024-11-10T17:23:06,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259446612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259446612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259446612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259446612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259446612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:06,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259446822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259446822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259446822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259446822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259446823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:06,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:06,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:06,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/801c72ebf2a44cd09fad84f91cbc6cb2 2024-11-10T17:23:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T17:23:06,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/c54cdc3116404d288d308ccbdad57f8e is 50, key is test_row_0/B:col10/1731259386482/Put/seqid=0 2024-11-10T17:23:06,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742096_1272 (size=12001) 2024-11-10T17:23:07,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:07,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259447132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259447132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259447132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259447132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259447132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:07,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:07,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:07,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/c54cdc3116404d288d308ccbdad57f8e 2024-11-10T17:23:07,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/34fd5f9870d947789c8c1e7a40a66c20 is 50, key is test_row_0/C:col10/1731259386482/Put/seqid=0 2024-11-10T17:23:07,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742097_1273 (size=12001) 2024-11-10T17:23:07,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/34fd5f9870d947789c8c1e7a40a66c20 2024-11-10T17:23:07,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/801c72ebf2a44cd09fad84f91cbc6cb2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/801c72ebf2a44cd09fad84f91cbc6cb2 2024-11-10T17:23:07,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/801c72ebf2a44cd09fad84f91cbc6cb2, entries=200, sequenceid=39, filesize=14.0 K 2024-11-10T17:23:07,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/c54cdc3116404d288d308ccbdad57f8e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c54cdc3116404d288d308ccbdad57f8e 2024-11-10T17:23:07,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c54cdc3116404d288d308ccbdad57f8e, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T17:23:07,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/34fd5f9870d947789c8c1e7a40a66c20 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/34fd5f9870d947789c8c1e7a40a66c20 2024-11-10T17:23:07,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/34fd5f9870d947789c8c1e7a40a66c20, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T17:23:07,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 84febc6fa98bac2754e6c1b07a211911 in 884ms, sequenceid=39, compaction requested=false 2024-11-10T17:23:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:07,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T17:23:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:07,469 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:23:07,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/5fad0141cd3f46858025f0c9041de9be is 50, key is test_row_0/A:col10/1731259386494/Put/seqid=0 2024-11-10T17:23:07,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742098_1274 (size=12001) 2024-11-10T17:23:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:07,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:07,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259447664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259447667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259447668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259447669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259447669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259447771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259447782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259447782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259447782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259447782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,880 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/5fad0141cd3f46858025f0c9041de9be 2024-11-10T17:23:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6618a3c9773d4de0a134de1f793ad775 is 50, key is test_row_0/B:col10/1731259386494/Put/seqid=0 2024-11-10T17:23:07,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742099_1275 (size=12001) 2024-11-10T17:23:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T17:23:07,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259447982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259447991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259447991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259447991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259447991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,293 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6618a3c9773d4de0a134de1f793ad775 2024-11-10T17:23:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259448291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/f65cf2575b5740bb8087e9a5f4558799 is 50, key is test_row_0/C:col10/1731259386494/Put/seqid=0 2024-11-10T17:23:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259448302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259448302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259448302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259448302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742100_1276 (size=12001) 2024-11-10T17:23:08,707 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/f65cf2575b5740bb8087e9a5f4558799 2024-11-10T17:23:08,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/5fad0141cd3f46858025f0c9041de9be as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5fad0141cd3f46858025f0c9041de9be 2024-11-10T17:23:08,716 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5fad0141cd3f46858025f0c9041de9be, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T17:23:08,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6618a3c9773d4de0a134de1f793ad775 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6618a3c9773d4de0a134de1f793ad775 2024-11-10T17:23:08,720 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6618a3c9773d4de0a134de1f793ad775, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T17:23:08,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/f65cf2575b5740bb8087e9a5f4558799 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f65cf2575b5740bb8087e9a5f4558799 2024-11-10T17:23:08,725 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f65cf2575b5740bb8087e9a5f4558799, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T17:23:08,725 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 84febc6fa98bac2754e6c1b07a211911 in 1256ms, sequenceid=50, compaction requested=true 2024-11-10T17:23:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-10T17:23:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-10T17:23:08,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T17:23:08,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9480 sec 2024-11-10T17:23:08,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.9570 sec 2024-11-10T17:23:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:08,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:08,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:23:08,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/545a125df40c47bdae7b7f9af7634f11 is 50, key is test_row_0/A:col10/1731259387668/Put/seqid=0 2024-11-10T17:23:08,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742101_1277 (size=14341) 2024-11-10T17:23:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259448815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259448816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259448816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259448817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259448818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259448921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259448921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259448921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259448921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:08,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259448931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259449131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259449131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259449131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259449131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259449141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/545a125df40c47bdae7b7f9af7634f11 2024-11-10T17:23:09,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b1c412c85e2e43608030e970c4984e8f is 50, key is test_row_0/B:col10/1731259387668/Put/seqid=0 2024-11-10T17:23:09,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742102_1278 (size=12001) 2024-11-10T17:23:09,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259449442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259449442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259449442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259449442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259449451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b1c412c85e2e43608030e970c4984e8f 2024-11-10T17:23:09,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/365310dd0c274d0db7a757380d21f2ae is 50, key is test_row_0/C:col10/1731259387668/Put/seqid=0 2024-11-10T17:23:09,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742103_1279 (size=12001) 2024-11-10T17:23:09,777 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T17:23:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T17:23:09,932 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:09,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-10T17:23:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T17:23:09,934 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:09,935 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:09,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:09,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259449951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259449951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259449961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259449961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259449962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T17:23:10,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/365310dd0c274d0db7a757380d21f2ae 2024-11-10T17:23:10,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/545a125df40c47bdae7b7f9af7634f11 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/545a125df40c47bdae7b7f9af7634f11 2024-11-10T17:23:10,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/545a125df40c47bdae7b7f9af7634f11, entries=200, sequenceid=76, filesize=14.0 K 2024-11-10T17:23:10,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b1c412c85e2e43608030e970c4984e8f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b1c412c85e2e43608030e970c4984e8f 2024-11-10T17:23:10,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b1c412c85e2e43608030e970c4984e8f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T17:23:10,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/365310dd0c274d0db7a757380d21f2ae as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/365310dd0c274d0db7a757380d21f2ae 2024-11-10T17:23:10,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/365310dd0c274d0db7a757380d21f2ae, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T17:23:10,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 84febc6fa98bac2754e6c1b07a211911 in 1257ms, sequenceid=76, compaction requested=true 2024-11-10T17:23:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:10,062 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:10,062 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:10,063 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:10,063 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/A is initiating minor compaction (all files) 2024-11-10T17:23:10,063 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/A in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:10,063 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:10,063 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c7f59cf4e5b645269991120d631b529a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/801c72ebf2a44cd09fad84f91cbc6cb2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5fad0141cd3f46858025f0c9041de9be, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/545a125df40c47bdae7b7f9af7634f11] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=51.4 K 2024-11-10T17:23:10,063 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/B is initiating minor compaction (all files) 2024-11-10T17:23:10,064 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/B in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:10,064 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/aa20ada6338749e1b2e9dd3f25e6d45f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c54cdc3116404d288d308ccbdad57f8e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6618a3c9773d4de0a134de1f793ad775, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b1c412c85e2e43608030e970c4984e8f] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=46.9 K 2024-11-10T17:23:10,064 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7f59cf4e5b645269991120d631b529a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731259385788 2024-11-10T17:23:10,064 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting aa20ada6338749e1b2e9dd3f25e6d45f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731259385788 2024-11-10T17:23:10,064 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 801c72ebf2a44cd09fad84f91cbc6cb2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731259385848 2024-11-10T17:23:10,064 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c54cdc3116404d288d308ccbdad57f8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731259385848 2024-11-10T17:23:10,065 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fad0141cd3f46858025f0c9041de9be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731259386494 2024-11-10T17:23:10,065 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6618a3c9773d4de0a134de1f793ad775, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731259386494 2024-11-10T17:23:10,065 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 545a125df40c47bdae7b7f9af7634f11, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259387663 2024-11-10T17:23:10,065 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c412c85e2e43608030e970c4984e8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259387663 2024-11-10T17:23:10,075 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:10,076 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/5b3486c8eec64c3ca56f0fa8cecb8c0f is 50, key is test_row_0/A:col10/1731259387668/Put/seqid=0 2024-11-10T17:23:10,078 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:10,079 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b33b64a3672043bd89c37389e326d631 is 50, key is test_row_0/B:col10/1731259387668/Put/seqid=0 2024-11-10T17:23:10,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-10T17:23:10,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:10,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T17:23:10,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742105_1281 (size=12139) 2024-11-10T17:23:10,102 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b33b64a3672043bd89c37389e326d631 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b33b64a3672043bd89c37389e326d631 2024-11-10T17:23:10,108 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/B of 84febc6fa98bac2754e6c1b07a211911 into b33b64a3672043bd89c37389e326d631(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:10,108 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:10,109 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/B, priority=12, startTime=1731259390062; duration=0sec 2024-11-10T17:23:10,109 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:10,109 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:B 2024-11-10T17:23:10,109 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:10,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/b9eecdb8616a442fb12cbe001055b3fd is 50, key is test_row_0/A:col10/1731259388817/Put/seqid=0 2024-11-10T17:23:10,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742104_1280 (size=12139) 2024-11-10T17:23:10,113 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:10,113 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/C is initiating minor compaction (all files) 2024-11-10T17:23:10,113 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/C in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:10,113 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/478cfb48fb1e44f8a312a2e75088f96b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/34fd5f9870d947789c8c1e7a40a66c20, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f65cf2575b5740bb8087e9a5f4558799, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/365310dd0c274d0db7a757380d21f2ae] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=46.9 K 2024-11-10T17:23:10,115 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 478cfb48fb1e44f8a312a2e75088f96b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731259385788 2024-11-10T17:23:10,116 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 34fd5f9870d947789c8c1e7a40a66c20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731259385848 2024-11-10T17:23:10,116 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f65cf2575b5740bb8087e9a5f4558799, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731259386494 2024-11-10T17:23:10,117 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 365310dd0c274d0db7a757380d21f2ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259387663 2024-11-10T17:23:10,118 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/5b3486c8eec64c3ca56f0fa8cecb8c0f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5b3486c8eec64c3ca56f0fa8cecb8c0f 2024-11-10T17:23:10,125 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/A of 84febc6fa98bac2754e6c1b07a211911 into 5b3486c8eec64c3ca56f0fa8cecb8c0f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:10,125 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:10,125 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/A, priority=12, startTime=1731259390062; duration=0sec 2024-11-10T17:23:10,125 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:10,125 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:A 2024-11-10T17:23:10,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742106_1282 (size=9657) 2024-11-10T17:23:10,128 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/b9eecdb8616a442fb12cbe001055b3fd 2024-11-10T17:23:10,139 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:10,139 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/3600f078661542e5a6cd8d3a8eeb1321 is 50, key is test_row_0/C:col10/1731259387668/Put/seqid=0 2024-11-10T17:23:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f904b13b79bb427886eca1dc7cbaf918 is 50, key is test_row_0/B:col10/1731259388817/Put/seqid=0 2024-11-10T17:23:10,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742108_1284 (size=9657) 2024-11-10T17:23:10,149 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f904b13b79bb427886eca1dc7cbaf918 2024-11-10T17:23:10,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/68853f1d737547daad0265318281b402 is 50, key is test_row_0/C:col10/1731259388817/Put/seqid=0 2024-11-10T17:23:10,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742107_1283 (size=12139) 2024-11-10T17:23:10,167 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/3600f078661542e5a6cd8d3a8eeb1321 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3600f078661542e5a6cd8d3a8eeb1321 2024-11-10T17:23:10,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742109_1285 (size=9657) 2024-11-10T17:23:10,172 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/C of 84febc6fa98bac2754e6c1b07a211911 into 3600f078661542e5a6cd8d3a8eeb1321(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:10,172 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:10,173 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/C, priority=12, startTime=1731259390062; duration=0sec 2024-11-10T17:23:10,173 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:10,173 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:C 2024-11-10T17:23:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T17:23:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T17:23:10,582 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/68853f1d737547daad0265318281b402 2024-11-10T17:23:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/b9eecdb8616a442fb12cbe001055b3fd as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b9eecdb8616a442fb12cbe001055b3fd 2024-11-10T17:23:10,592 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b9eecdb8616a442fb12cbe001055b3fd, entries=100, sequenceid=86, filesize=9.4 K 2024-11-10T17:23:10,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f904b13b79bb427886eca1dc7cbaf918 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f904b13b79bb427886eca1dc7cbaf918 2024-11-10T17:23:10,597 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f904b13b79bb427886eca1dc7cbaf918, entries=100, sequenceid=86, filesize=9.4 K 2024-11-10T17:23:10,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/68853f1d737547daad0265318281b402 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/68853f1d737547daad0265318281b402 2024-11-10T17:23:10,604 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/68853f1d737547daad0265318281b402, entries=100, sequenceid=86, filesize=9.4 K 2024-11-10T17:23:10,605 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 84febc6fa98bac2754e6c1b07a211911 in 518ms, sequenceid=86, compaction requested=false 2024-11-10T17:23:10,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:10,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:10,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-10T17:23:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-10T17:23:10,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T17:23:10,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 671 msec 2024-11-10T17:23:10,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 676 msec 2024-11-10T17:23:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:10,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:10,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:23:10,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/35ebbc266c0744dbbbbff1d3a333e2e7 is 50, key is test_row_0/A:col10/1731259390972/Put/seqid=0 2024-11-10T17:23:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259450995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259450997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259450998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259451000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259451001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742110_1286 (size=14341) 2024-11-10T17:23:11,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/35ebbc266c0744dbbbbff1d3a333e2e7 2024-11-10T17:23:11,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6154220da2e64be79ed4175998e9d73b is 50, key is test_row_0/B:col10/1731259390972/Put/seqid=0 2024-11-10T17:23:11,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742111_1287 (size=12001) 2024-11-10T17:23:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T17:23:11,072 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:11,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-10T17:23:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T17:23:11,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:11,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:11,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:11,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259451111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259451111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259451112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259451112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259451112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T17:23:11,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T17:23:11,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:11,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259451322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259451322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259451322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259451322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259451322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T17:23:11,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:11,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T17:23:11,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6154220da2e64be79ed4175998e9d73b 2024-11-10T17:23:11,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/5c687ed9b2f249fdbc4a3e3ec1106ec0 is 50, key is test_row_0/C:col10/1731259390972/Put/seqid=0 2024-11-10T17:23:11,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742112_1288 (size=12001) 2024-11-10T17:23:11,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T17:23:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259451631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259451632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259451632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259451632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259451632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:11,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T17:23:11,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:11,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T17:23:11,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/5c687ed9b2f249fdbc4a3e3ec1106ec0 2024-11-10T17:23:11,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/35ebbc266c0744dbbbbff1d3a333e2e7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/35ebbc266c0744dbbbbff1d3a333e2e7 2024-11-10T17:23:11,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/35ebbc266c0744dbbbbff1d3a333e2e7, entries=200, sequenceid=100, filesize=14.0 K 2024-11-10T17:23:11,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6154220da2e64be79ed4175998e9d73b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6154220da2e64be79ed4175998e9d73b 2024-11-10T17:23:11,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T17:23:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:11,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6154220da2e64be79ed4175998e9d73b, entries=150, sequenceid=100, filesize=11.7 K 2024-11-10T17:23:11,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/5c687ed9b2f249fdbc4a3e3ec1106ec0 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/5c687ed9b2f249fdbc4a3e3ec1106ec0 2024-11-10T17:23:11,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/5c687ed9b2f249fdbc4a3e3ec1106ec0, entries=150, sequenceid=100, filesize=11.7 K 2024-11-10T17:23:11,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 84febc6fa98bac2754e6c1b07a211911 in 878ms, sequenceid=100, compaction requested=true 2024-11-10T17:23:11,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:11,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:11,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:11,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:11,853 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:11,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:11,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:11,853 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:11,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:11,854 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36137 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:11,854 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:11,854 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/A is initiating minor compaction (all files) 2024-11-10T17:23:11,854 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/B is initiating minor compaction (all files) 2024-11-10T17:23:11,854 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/B in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,854 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/A in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,854 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b33b64a3672043bd89c37389e326d631, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f904b13b79bb427886eca1dc7cbaf918, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6154220da2e64be79ed4175998e9d73b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=33.0 K 2024-11-10T17:23:11,854 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5b3486c8eec64c3ca56f0fa8cecb8c0f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b9eecdb8616a442fb12cbe001055b3fd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/35ebbc266c0744dbbbbff1d3a333e2e7] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=35.3 K 2024-11-10T17:23:11,855 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b33b64a3672043bd89c37389e326d631, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259387663 2024-11-10T17:23:11,855 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b3486c8eec64c3ca56f0fa8cecb8c0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259387663 2024-11-10T17:23:11,855 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9eecdb8616a442fb12cbe001055b3fd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731259388814 2024-11-10T17:23:11,855 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f904b13b79bb427886eca1dc7cbaf918, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731259388814 2024-11-10T17:23:11,855 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6154220da2e64be79ed4175998e9d73b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731259390972 2024-11-10T17:23:11,855 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ebbc266c0744dbbbbff1d3a333e2e7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731259390968 2024-11-10T17:23:11,871 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:11,872 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/a526cb85d3314181b0ef02ac8a0c42f6 is 50, key is test_row_0/A:col10/1731259390972/Put/seqid=0 2024-11-10T17:23:11,874 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:11,875 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/af18ab68d3ee47db8259895f44ec2acb is 50, key is test_row_0/B:col10/1731259390972/Put/seqid=0 2024-11-10T17:23:11,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742113_1289 (size=12241) 2024-11-10T17:23:11,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742114_1290 (size=12241) 2024-11-10T17:23:11,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T17:23:11,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:11,995 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:23:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/baf863462ef046d19292dd4de65cadcb is 50, key is test_row_0/A:col10/1731259390999/Put/seqid=0 2024-11-10T17:23:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742115_1291 (size=12001) 2024-11-10T17:23:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:12,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:12,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259452147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259452151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259452152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259452154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259452154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T17:23:12,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259452261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259452262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259452262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259452262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259452262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,303 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/a526cb85d3314181b0ef02ac8a0c42f6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/a526cb85d3314181b0ef02ac8a0c42f6 2024-11-10T17:23:12,303 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/af18ab68d3ee47db8259895f44ec2acb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/af18ab68d3ee47db8259895f44ec2acb 2024-11-10T17:23:12,308 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/A of 84febc6fa98bac2754e6c1b07a211911 into a526cb85d3314181b0ef02ac8a0c42f6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:12,308 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/B of 84febc6fa98bac2754e6c1b07a211911 into af18ab68d3ee47db8259895f44ec2acb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:12,308 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:12,308 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:12,308 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/A, priority=13, startTime=1731259391852; duration=0sec 2024-11-10T17:23:12,308 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/B, priority=13, startTime=1731259391853; duration=0sec 2024-11-10T17:23:12,308 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:12,308 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:A 2024-11-10T17:23:12,308 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:12,308 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:B 2024-11-10T17:23:12,308 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:12,310 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:12,310 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/C is initiating minor compaction (all files) 2024-11-10T17:23:12,310 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/C in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:12,310 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3600f078661542e5a6cd8d3a8eeb1321, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/68853f1d737547daad0265318281b402, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/5c687ed9b2f249fdbc4a3e3ec1106ec0] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=33.0 K 2024-11-10T17:23:12,310 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3600f078661542e5a6cd8d3a8eeb1321, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731259387663 2024-11-10T17:23:12,310 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68853f1d737547daad0265318281b402, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731259388814 2024-11-10T17:23:12,311 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c687ed9b2f249fdbc4a3e3ec1106ec0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731259390972 2024-11-10T17:23:12,317 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:12,317 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/67b63b3ddec94bf69005c9bac048b8eb is 50, key is test_row_0/C:col10/1731259390972/Put/seqid=0 2024-11-10T17:23:12,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742116_1292 (size=12241) 2024-11-10T17:23:12,404 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/baf863462ef046d19292dd4de65cadcb 2024-11-10T17:23:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a is 50, key is test_row_0/B:col10/1731259390999/Put/seqid=0 2024-11-10T17:23:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742117_1293 (size=12001) 2024-11-10T17:23:12,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259452472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259452472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259452472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259452472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259452472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,726 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/67b63b3ddec94bf69005c9bac048b8eb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/67b63b3ddec94bf69005c9bac048b8eb 2024-11-10T17:23:12,745 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/C of 84febc6fa98bac2754e6c1b07a211911 into 67b63b3ddec94bf69005c9bac048b8eb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:12,745 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:12,745 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/C, priority=13, startTime=1731259391853; duration=0sec 2024-11-10T17:23:12,745 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:12,745 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:C 2024-11-10T17:23:12,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259452781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259452782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259452782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259452782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259452782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:12,816 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a 2024-11-10T17:23:12,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/e78146041e1947f4b26ed595f9d8083d is 50, key is test_row_0/C:col10/1731259390999/Put/seqid=0 2024-11-10T17:23:12,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742118_1294 (size=12001) 2024-11-10T17:23:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T17:23:13,229 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/e78146041e1947f4b26ed595f9d8083d 2024-11-10T17:23:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/baf863462ef046d19292dd4de65cadcb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/baf863462ef046d19292dd4de65cadcb 2024-11-10T17:23:13,238 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/baf863462ef046d19292dd4de65cadcb, entries=150, sequenceid=125, filesize=11.7 K 2024-11-10T17:23:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a 2024-11-10T17:23:13,245 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a, entries=150, sequenceid=125, filesize=11.7 K 2024-11-10T17:23:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/e78146041e1947f4b26ed595f9d8083d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e78146041e1947f4b26ed595f9d8083d 2024-11-10T17:23:13,249 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e78146041e1947f4b26ed595f9d8083d, entries=150, sequenceid=125, filesize=11.7 K 2024-11-10T17:23:13,250 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 84febc6fa98bac2754e6c1b07a211911 in 1255ms, sequenceid=125, compaction requested=false 2024-11-10T17:23:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:13,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:13,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-10T17:23:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-10T17:23:13,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-10T17:23:13,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-11-10T17:23:13,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1820 sec 2024-11-10T17:23:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:13,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:13,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:23:13,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/fc9383bea90744dda27809e9367f9a5b is 50, key is test_row_0/A:col10/1731259393294/Put/seqid=0 2024-11-10T17:23:13,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742119_1295 (size=12151) 2024-11-10T17:23:13,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/fc9383bea90744dda27809e9367f9a5b 2024-11-10T17:23:13,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/32af7323ab8649ae9f6471efefb3d53c is 50, key is test_row_0/B:col10/1731259393294/Put/seqid=0 2024-11-10T17:23:13,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742120_1296 (size=12151) 2024-11-10T17:23:13,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/32af7323ab8649ae9f6471efefb3d53c 2024-11-10T17:23:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259453317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259453317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259453318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259453319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259453323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/e64198cc21d64a72acbd5f0190fa1e56 is 50, key is test_row_0/C:col10/1731259393294/Put/seqid=0 2024-11-10T17:23:13,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742121_1297 (size=12151) 2024-11-10T17:23:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259453431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259453431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259453431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259453432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259453432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259453641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259453641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259453641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259453641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259453642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/e64198cc21d64a72acbd5f0190fa1e56 2024-11-10T17:23:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/fc9383bea90744dda27809e9367f9a5b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/fc9383bea90744dda27809e9367f9a5b 2024-11-10T17:23:13,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/fc9383bea90744dda27809e9367f9a5b, entries=150, sequenceid=142, filesize=11.9 K 2024-11-10T17:23:13,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/32af7323ab8649ae9f6471efefb3d53c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/32af7323ab8649ae9f6471efefb3d53c 2024-11-10T17:23:13,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/32af7323ab8649ae9f6471efefb3d53c, entries=150, sequenceid=142, filesize=11.9 K 2024-11-10T17:23:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/e64198cc21d64a72acbd5f0190fa1e56 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e64198cc21d64a72acbd5f0190fa1e56 2024-11-10T17:23:13,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e64198cc21d64a72acbd5f0190fa1e56, entries=150, sequenceid=142, filesize=11.9 K 2024-11-10T17:23:13,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 84febc6fa98bac2754e6c1b07a211911 in 458ms, sequenceid=142, compaction requested=true 2024-11-10T17:23:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:13,752 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:13,752 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:13,753 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:13,753 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/B is initiating minor compaction (all files) 2024-11-10T17:23:13,753 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:13,753 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/B in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:13,753 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/A is initiating minor compaction (all files) 2024-11-10T17:23:13,753 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/A in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:13,753 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/af18ab68d3ee47db8259895f44ec2acb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/32af7323ab8649ae9f6471efefb3d53c] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=35.5 K 2024-11-10T17:23:13,753 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/a526cb85d3314181b0ef02ac8a0c42f6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/baf863462ef046d19292dd4de65cadcb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/fc9383bea90744dda27809e9367f9a5b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=35.5 K 2024-11-10T17:23:13,754 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting af18ab68d3ee47db8259895f44ec2acb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731259390972 2024-11-10T17:23:13,754 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting a526cb85d3314181b0ef02ac8a0c42f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731259390972 2024-11-10T17:23:13,754 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b1e2c6b2fb4d7abf4c05d6ff96c37a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731259390993 2024-11-10T17:23:13,754 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf863462ef046d19292dd4de65cadcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731259390993 2024-11-10T17:23:13,754 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 32af7323ab8649ae9f6471efefb3d53c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731259392152 2024-11-10T17:23:13,755 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc9383bea90744dda27809e9367f9a5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731259392152 2024-11-10T17:23:13,766 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:13,766 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:13,767 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/b7003af503d14bf0a7e9e28ff3abc33a is 50, key is test_row_0/A:col10/1731259393294/Put/seqid=0 2024-11-10T17:23:13,767 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/77d748e3c68b4d2fbfb328d8a6f49e3a is 50, key is test_row_0/B:col10/1731259393294/Put/seqid=0 2024-11-10T17:23:13,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742122_1298 (size=12493) 2024-11-10T17:23:13,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742123_1299 (size=12493) 2024-11-10T17:23:13,780 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/77d748e3c68b4d2fbfb328d8a6f49e3a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/77d748e3c68b4d2fbfb328d8a6f49e3a 2024-11-10T17:23:13,785 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/B of 84febc6fa98bac2754e6c1b07a211911 into 77d748e3c68b4d2fbfb328d8a6f49e3a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:13,785 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:13,785 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/B, priority=13, startTime=1731259393752; duration=0sec 2024-11-10T17:23:13,785 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:13,785 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:B 2024-11-10T17:23:13,785 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:13,786 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:13,786 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/C is initiating minor compaction (all files) 2024-11-10T17:23:13,786 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/C in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:13,786 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/67b63b3ddec94bf69005c9bac048b8eb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e78146041e1947f4b26ed595f9d8083d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e64198cc21d64a72acbd5f0190fa1e56] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=35.5 K 2024-11-10T17:23:13,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 67b63b3ddec94bf69005c9bac048b8eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731259390972 2024-11-10T17:23:13,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e78146041e1947f4b26ed595f9d8083d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731259390993 2024-11-10T17:23:13,788 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e64198cc21d64a72acbd5f0190fa1e56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731259392152 2024-11-10T17:23:13,794 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:13,795 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/908d8ba695554947b19ec52881ee584e is 50, key is test_row_0/C:col10/1731259393294/Put/seqid=0 2024-11-10T17:23:13,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742124_1300 (size=12493) 2024-11-10T17:23:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:13,955 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:13,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T17:23:13,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/7b8b6c6742474c81aaa13ecaf920aad1 is 50, key is test_row_0/A:col10/1731259393316/Put/seqid=0 2024-11-10T17:23:13,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742125_1301 (size=12151) 2024-11-10T17:23:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259453963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259453992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259453992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259453992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259453993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:13,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259454102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259454102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259454102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259454102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259454102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,177 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/b7003af503d14bf0a7e9e28ff3abc33a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b7003af503d14bf0a7e9e28ff3abc33a 2024-11-10T17:23:14,182 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/A of 84febc6fa98bac2754e6c1b07a211911 into b7003af503d14bf0a7e9e28ff3abc33a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:14,182 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:14,182 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/A, priority=13, startTime=1731259393752; duration=0sec 2024-11-10T17:23:14,182 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:14,182 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:A 2024-11-10T17:23:14,205 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/908d8ba695554947b19ec52881ee584e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/908d8ba695554947b19ec52881ee584e 2024-11-10T17:23:14,210 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/C of 84febc6fa98bac2754e6c1b07a211911 into 908d8ba695554947b19ec52881ee584e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:14,210 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:14,210 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/C, priority=13, startTime=1731259393752; duration=0sec 2024-11-10T17:23:14,210 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:14,210 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:C 2024-11-10T17:23:14,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259454312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259454312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259454312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259454312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259454312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/7b8b6c6742474c81aaa13ecaf920aad1 2024-11-10T17:23:14,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f26d6fcb26bc4e5ebd178605981b55cf is 50, key is test_row_0/B:col10/1731259393316/Put/seqid=0 2024-11-10T17:23:14,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742126_1302 (size=12151) 2024-11-10T17:23:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:14,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f26d6fcb26bc4e5ebd178605981b55cf 2024-11-10T17:23:14,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/7ffe61e7c5b94f88b9467bb85305c112 is 50, key is test_row_0/C:col10/1731259393316/Put/seqid=0 2024-11-10T17:23:14,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742127_1303 (size=12151) 2024-11-10T17:23:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259455132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259455132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259455132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:15,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259455141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259455141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:15,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/7ffe61e7c5b94f88b9467bb85305c112 2024-11-10T17:23:15,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/7b8b6c6742474c81aaa13ecaf920aad1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/7b8b6c6742474c81aaa13ecaf920aad1 2024-11-10T17:23:15,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/7b8b6c6742474c81aaa13ecaf920aad1, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T17:23:15,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f26d6fcb26bc4e5ebd178605981b55cf as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f26d6fcb26bc4e5ebd178605981b55cf 2024-11-10T17:23:15,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f26d6fcb26bc4e5ebd178605981b55cf, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T17:23:15,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/7ffe61e7c5b94f88b9467bb85305c112 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/7ffe61e7c5b94f88b9467bb85305c112 2024-11-10T17:23:15,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/7ffe61e7c5b94f88b9467bb85305c112, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T17:23:15,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 84febc6fa98bac2754e6c1b07a211911 in 1252ms, sequenceid=166, compaction requested=false 2024-11-10T17:23:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T17:23:15,232 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:15,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-10T17:23:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T17:23:15,234 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:15,235 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:15,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T17:23:15,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T17:23:15,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:15,388 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:23:15,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/389398a85f6c4fa4b7db390c7a98b7ea is 50, key is test_row_0/A:col10/1731259393991/Put/seqid=0 2024-11-10T17:23:15,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742128_1304 (size=12151) 2024-11-10T17:23:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T17:23:15,799 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/389398a85f6c4fa4b7db390c7a98b7ea 2024-11-10T17:23:15,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/d985e9d0e77a44689b90c1dddfa64b27 is 50, key is test_row_0/B:col10/1731259393991/Put/seqid=0 2024-11-10T17:23:15,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742129_1305 (size=12151) 2024-11-10T17:23:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T17:23:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:16,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:16,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259456165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259456166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259456168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259456169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259456170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,225 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/d985e9d0e77a44689b90c1dddfa64b27 2024-11-10T17:23:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8 is 50, key is test_row_0/C:col10/1731259393991/Put/seqid=0 2024-11-10T17:23:16,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742130_1306 (size=12151) 2024-11-10T17:23:16,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259456281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259456281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259456281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259456282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259456282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T17:23:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259456491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259456492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259456492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259456492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259456492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,637 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8 2024-11-10T17:23:16,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/389398a85f6c4fa4b7db390c7a98b7ea as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/389398a85f6c4fa4b7db390c7a98b7ea 2024-11-10T17:23:16,649 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/389398a85f6c4fa4b7db390c7a98b7ea, entries=150, sequenceid=181, filesize=11.9 K 2024-11-10T17:23:16,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/d985e9d0e77a44689b90c1dddfa64b27 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d985e9d0e77a44689b90c1dddfa64b27 2024-11-10T17:23:16,654 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d985e9d0e77a44689b90c1dddfa64b27, entries=150, sequenceid=181, filesize=11.9 K 2024-11-10T17:23:16,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8 2024-11-10T17:23:16,661 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8, entries=150, sequenceid=181, filesize=11.9 K 2024-11-10T17:23:16,662 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 84febc6fa98bac2754e6c1b07a211911 in 1274ms, sequenceid=181, compaction requested=true 2024-11-10T17:23:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-10T17:23:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-10T17:23:16,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T17:23:16,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-10T17:23:16,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4360 sec 2024-11-10T17:23:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:16,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:16,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:23:16,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d55aff33795d48839af66b5ae183c3dc is 50, key is test_row_0/A:col10/1731259396162/Put/seqid=0 2024-11-10T17:23:16,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259456814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259456815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259456816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259456816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259456817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742131_1307 (size=12151) 2024-11-10T17:23:16,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d55aff33795d48839af66b5ae183c3dc 2024-11-10T17:23:16,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b10e5325a6d441e5923a86b73147349b is 50, key is test_row_0/B:col10/1731259396162/Put/seqid=0 2024-11-10T17:23:16,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742132_1308 (size=12151) 2024-11-10T17:23:16,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b10e5325a6d441e5923a86b73147349b 2024-11-10T17:23:16,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/999b36003ac84d30b213670b306675a5 is 50, key is test_row_0/C:col10/1731259396162/Put/seqid=0 2024-11-10T17:23:16,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742133_1309 (size=12151) 2024-11-10T17:23:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259456932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259456932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259456933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259456933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259456933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259457141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259457141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259457142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259457142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259457144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/999b36003ac84d30b213670b306675a5 2024-11-10T17:23:17,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d55aff33795d48839af66b5ae183c3dc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d55aff33795d48839af66b5ae183c3dc 2024-11-10T17:23:17,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d55aff33795d48839af66b5ae183c3dc, entries=150, sequenceid=206, filesize=11.9 K 2024-11-10T17:23:17,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/b10e5325a6d441e5923a86b73147349b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b10e5325a6d441e5923a86b73147349b 2024-11-10T17:23:17,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b10e5325a6d441e5923a86b73147349b, entries=150, sequenceid=206, filesize=11.9 K 2024-11-10T17:23:17,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/999b36003ac84d30b213670b306675a5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/999b36003ac84d30b213670b306675a5 2024-11-10T17:23:17,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/999b36003ac84d30b213670b306675a5, entries=150, sequenceid=206, filesize=11.9 K 2024-11-10T17:23:17,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 84febc6fa98bac2754e6c1b07a211911 in 485ms, sequenceid=206, compaction requested=true 2024-11-10T17:23:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:17,290 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:17,290 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:17,292 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:17,292 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/B is initiating minor compaction (all files) 2024-11-10T17:23:17,292 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/B in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,293 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/77d748e3c68b4d2fbfb328d8a6f49e3a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f26d6fcb26bc4e5ebd178605981b55cf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d985e9d0e77a44689b90c1dddfa64b27, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b10e5325a6d441e5923a86b73147349b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=47.8 K 2024-11-10T17:23:17,293 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:17,293 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/A is initiating minor compaction (all files) 2024-11-10T17:23:17,293 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/A in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,293 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b7003af503d14bf0a7e9e28ff3abc33a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/7b8b6c6742474c81aaa13ecaf920aad1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/389398a85f6c4fa4b7db390c7a98b7ea, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d55aff33795d48839af66b5ae183c3dc] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=47.8 K 2024-11-10T17:23:17,293 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d748e3c68b4d2fbfb328d8a6f49e3a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731259392152 2024-11-10T17:23:17,293 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7003af503d14bf0a7e9e28ff3abc33a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731259392152 2024-11-10T17:23:17,294 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f26d6fcb26bc4e5ebd178605981b55cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731259393316 2024-11-10T17:23:17,294 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b8b6c6742474c81aaa13ecaf920aad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731259393316 2024-11-10T17:23:17,294 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d985e9d0e77a44689b90c1dddfa64b27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731259393962 2024-11-10T17:23:17,294 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389398a85f6c4fa4b7db390c7a98b7ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731259393962 2024-11-10T17:23:17,295 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b10e5325a6d441e5923a86b73147349b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259396162 2024-11-10T17:23:17,295 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55aff33795d48839af66b5ae183c3dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259396162 2024-11-10T17:23:17,310 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:17,311 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6ff66f143e5249b1889237020c7059e3 is 50, key is test_row_0/B:col10/1731259396162/Put/seqid=0 2024-11-10T17:23:17,312 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:17,313 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/5ce2bc44eea04b6ba3a141c01d0dae52 is 50, key is test_row_0/A:col10/1731259396162/Put/seqid=0 2024-11-10T17:23:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742134_1310 (size=12629) 2024-11-10T17:23:17,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742135_1311 (size=12629) 2024-11-10T17:23:17,324 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/6ff66f143e5249b1889237020c7059e3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6ff66f143e5249b1889237020c7059e3 2024-11-10T17:23:17,326 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/5ce2bc44eea04b6ba3a141c01d0dae52 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5ce2bc44eea04b6ba3a141c01d0dae52 2024-11-10T17:23:17,332 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/B of 84febc6fa98bac2754e6c1b07a211911 into 6ff66f143e5249b1889237020c7059e3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:17,332 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:17,332 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/B, priority=12, startTime=1731259397290; duration=0sec 2024-11-10T17:23:17,332 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/A of 84febc6fa98bac2754e6c1b07a211911 into 5ce2bc44eea04b6ba3a141c01d0dae52(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:17,332 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:17,332 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:B 2024-11-10T17:23:17,332 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:17,332 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/A, priority=12, startTime=1731259397290; duration=0sec 2024-11-10T17:23:17,332 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:17,332 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:17,332 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:A 2024-11-10T17:23:17,334 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:17,334 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/C is initiating minor compaction (all files) 2024-11-10T17:23:17,334 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/C in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,334 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/908d8ba695554947b19ec52881ee584e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/7ffe61e7c5b94f88b9467bb85305c112, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/999b36003ac84d30b213670b306675a5] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=47.8 K 2024-11-10T17:23:17,335 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 908d8ba695554947b19ec52881ee584e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731259392152 2024-11-10T17:23:17,335 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ffe61e7c5b94f88b9467bb85305c112, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731259393316 2024-11-10T17:23:17,336 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting dc9f8fb9a90f4f5fb3ad639bc361f8c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731259393962 2024-11-10T17:23:17,336 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 999b36003ac84d30b213670b306675a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259396162 2024-11-10T17:23:17,351 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:17,352 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/91d5849b3c9c47f1b27602adcd3af549 is 50, key is test_row_0/C:col10/1731259396162/Put/seqid=0 2024-11-10T17:23:17,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742136_1312 (size=12629) 2024-11-10T17:23:17,369 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/91d5849b3c9c47f1b27602adcd3af549 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/91d5849b3c9c47f1b27602adcd3af549 2024-11-10T17:23:17,374 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/C of 84febc6fa98bac2754e6c1b07a211911 into 91d5849b3c9c47f1b27602adcd3af549(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:17,374 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:17,374 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/C, priority=12, startTime=1731259397290; duration=0sec 2024-11-10T17:23:17,374 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:17,374 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:C 2024-11-10T17:23:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T17:23:17,382 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:17,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T17:23:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T17:23:17,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:17,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:17,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:17,454 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:17,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:23:17,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/1bb476df631848368aaf8d2367577359 is 50, key is test_row_0/A:col10/1731259397454/Put/seqid=0 2024-11-10T17:23:17,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742137_1313 (size=12151) 2024-11-10T17:23:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259457477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259457478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259457480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259457482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259457483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T17:23:17,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:17,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:17,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259457591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259457592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259457592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259457592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259457592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T17:23:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259457802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259457802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259457802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259457802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259457802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:17,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:17,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:17,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/1bb476df631848368aaf8d2367577359 2024-11-10T17:23:17,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/963cf6de63c448d5a32c54e7234bf722 is 50, key is test_row_0/B:col10/1731259397454/Put/seqid=0 2024-11-10T17:23:17,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742138_1314 (size=12151) 2024-11-10T17:23:17,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:17,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T17:23:18,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259458112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259458112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259458112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259458112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259458112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/963cf6de63c448d5a32c54e7234bf722 2024-11-10T17:23:18,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/71c4427093c04c139b40688c29788b55 is 50, key is test_row_0/C:col10/1731259397454/Put/seqid=0 2024-11-10T17:23:18,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742139_1315 (size=12151) 2024-11-10T17:23:18,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:18,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:18,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T17:23:18,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:18,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:18,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:18,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259458622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259458622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259458622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259458622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259458622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:18,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/71c4427093c04c139b40688c29788b55 2024-11-10T17:23:18,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/1bb476df631848368aaf8d2367577359 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/1bb476df631848368aaf8d2367577359 2024-11-10T17:23:18,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/1bb476df631848368aaf8d2367577359, entries=150, sequenceid=223, filesize=11.9 K 2024-11-10T17:23:18,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/963cf6de63c448d5a32c54e7234bf722 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/963cf6de63c448d5a32c54e7234bf722 2024-11-10T17:23:18,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/963cf6de63c448d5a32c54e7234bf722, entries=150, sequenceid=223, filesize=11.9 K 2024-11-10T17:23:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/71c4427093c04c139b40688c29788b55 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/71c4427093c04c139b40688c29788b55 2024-11-10T17:23:18,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/71c4427093c04c139b40688c29788b55, entries=150, sequenceid=223, filesize=11.9 K 2024-11-10T17:23:18,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 84febc6fa98bac2754e6c1b07a211911 in 1254ms, sequenceid=223, compaction requested=false 2024-11-10T17:23:18,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:18,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T17:23:18,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:18,765 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T17:23:18,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d913771841214d48a394577d177e7c66 is 50, key is test_row_0/A:col10/1731259397476/Put/seqid=0 2024-11-10T17:23:18,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742140_1316 (size=12151) 2024-11-10T17:23:19,174 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d913771841214d48a394577d177e7c66 2024-11-10T17:23:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/4143278ad83d4e62b92bfe59975a74cf is 50, key is test_row_0/B:col10/1731259397476/Put/seqid=0 2024-11-10T17:23:19,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742141_1317 (size=12151) 2024-11-10T17:23:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T17:23:19,587 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/4143278ad83d4e62b92bfe59975a74cf 2024-11-10T17:23:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/467f52b6c8b641daa6d838b401bff602 is 50, key is test_row_0/C:col10/1731259397476/Put/seqid=0 2024-11-10T17:23:19,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742142_1318 (size=12151) 2024-11-10T17:23:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:19,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:19,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259459647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259459656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259459675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259459675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259459675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259459781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259459781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259459781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259459792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259459792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259459991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259459991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259459991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:19,998 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/467f52b6c8b641daa6d838b401bff602 2024-11-10T17:23:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d913771841214d48a394577d177e7c66 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d913771841214d48a394577d177e7c66 2024-11-10T17:23:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259460002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259460002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,007 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d913771841214d48a394577d177e7c66, entries=150, sequenceid=245, filesize=11.9 K 2024-11-10T17:23:20,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/4143278ad83d4e62b92bfe59975a74cf as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4143278ad83d4e62b92bfe59975a74cf 2024-11-10T17:23:20,012 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4143278ad83d4e62b92bfe59975a74cf, entries=150, sequenceid=245, filesize=11.9 K 2024-11-10T17:23:20,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/467f52b6c8b641daa6d838b401bff602 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/467f52b6c8b641daa6d838b401bff602 2024-11-10T17:23:20,016 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/467f52b6c8b641daa6d838b401bff602, entries=150, sequenceid=245, filesize=11.9 K 2024-11-10T17:23:20,018 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 84febc6fa98bac2754e6c1b07a211911 in 1253ms, sequenceid=245, compaction requested=true 2024-11-10T17:23:20,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:20,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:20,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T17:23:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T17:23:20,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T17:23:20,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6340 sec 2024-11-10T17:23:20,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6390 sec 2024-11-10T17:23:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:20,304 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:20,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T17:23:20,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/02f648e57556493196aa501da26465b5 is 50, key is test_row_0/A:col10/1731259400303/Put/seqid=0 2024-11-10T17:23:20,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742143_1319 (size=14741) 2024-11-10T17:23:20,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/02f648e57556493196aa501da26465b5 2024-11-10T17:23:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259460324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259460325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259460326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259460330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259460331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/23fdaf0ce6824cf89ed0cc1f2919694e is 50, key is test_row_0/B:col10/1731259400303/Put/seqid=0 2024-11-10T17:23:20,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742144_1320 (size=12301) 2024-11-10T17:23:20,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259460442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259460442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259460442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259460442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259460442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259460652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259460652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259460652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259460652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259460652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/23fdaf0ce6824cf89ed0cc1f2919694e 2024-11-10T17:23:20,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/70ee68fb697d4c15a7c724a35030fe2e is 50, key is test_row_0/C:col10/1731259400303/Put/seqid=0 2024-11-10T17:23:20,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742145_1321 (size=12301) 2024-11-10T17:23:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259460962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259460962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259460962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259460962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259460962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:20,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/70ee68fb697d4c15a7c724a35030fe2e 2024-11-10T17:23:21,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/02f648e57556493196aa501da26465b5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/02f648e57556493196aa501da26465b5 2024-11-10T17:23:21,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/02f648e57556493196aa501da26465b5, entries=200, sequenceid=262, filesize=14.4 K 2024-11-10T17:23:21,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/23fdaf0ce6824cf89ed0cc1f2919694e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/23fdaf0ce6824cf89ed0cc1f2919694e 2024-11-10T17:23:21,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/23fdaf0ce6824cf89ed0cc1f2919694e, entries=150, sequenceid=262, filesize=12.0 K 2024-11-10T17:23:21,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/70ee68fb697d4c15a7c724a35030fe2e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/70ee68fb697d4c15a7c724a35030fe2e 2024-11-10T17:23:21,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/70ee68fb697d4c15a7c724a35030fe2e, entries=150, sequenceid=262, filesize=12.0 K 2024-11-10T17:23:21,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 84febc6fa98bac2754e6c1b07a211911 in 874ms, sequenceid=262, compaction requested=true 2024-11-10T17:23:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:21,178 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:21,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:21,178 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:21,180 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51672 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:21,180 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/A is initiating minor compaction (all files) 2024-11-10T17:23:21,180 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/A in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:21,180 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5ce2bc44eea04b6ba3a141c01d0dae52, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/1bb476df631848368aaf8d2367577359, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d913771841214d48a394577d177e7c66, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/02f648e57556493196aa501da26465b5] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=50.5 K 2024-11-10T17:23:21,180 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:21,180 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/B is initiating minor compaction (all files) 2024-11-10T17:23:21,180 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/B in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:21,180 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6ff66f143e5249b1889237020c7059e3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/963cf6de63c448d5a32c54e7234bf722, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4143278ad83d4e62b92bfe59975a74cf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/23fdaf0ce6824cf89ed0cc1f2919694e] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=48.1 K 2024-11-10T17:23:21,181 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce2bc44eea04b6ba3a141c01d0dae52, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259396162 2024-11-10T17:23:21,181 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ff66f143e5249b1889237020c7059e3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259396162 2024-11-10T17:23:21,181 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb476df631848368aaf8d2367577359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731259397452 2024-11-10T17:23:21,181 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 963cf6de63c448d5a32c54e7234bf722, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731259397452 2024-11-10T17:23:21,181 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting d913771841214d48a394577d177e7c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731259397475 2024-11-10T17:23:21,182 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4143278ad83d4e62b92bfe59975a74cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731259397475 2024-11-10T17:23:21,182 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f648e57556493196aa501da26465b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731259399646 2024-11-10T17:23:21,182 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fdaf0ce6824cf89ed0cc1f2919694e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731259399661 2024-11-10T17:23:21,192 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:21,193 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/23c4a0b0cff9462cb02409bb9a9326d1 is 50, key is test_row_0/A:col10/1731259400303/Put/seqid=0 2024-11-10T17:23:21,197 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:21,197 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/3325d21cccdf4b1687f74d44498e7588 is 50, key is test_row_0/B:col10/1731259400303/Put/seqid=0 2024-11-10T17:23:21,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742146_1322 (size=12915) 2024-11-10T17:23:21,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742147_1323 (size=12915) 2024-11-10T17:23:21,227 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/23c4a0b0cff9462cb02409bb9a9326d1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/23c4a0b0cff9462cb02409bb9a9326d1 2024-11-10T17:23:21,236 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/A of 84febc6fa98bac2754e6c1b07a211911 into 23c4a0b0cff9462cb02409bb9a9326d1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:21,236 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:21,236 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/A, priority=12, startTime=1731259401178; duration=0sec 2024-11-10T17:23:21,237 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:21,237 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:A 2024-11-10T17:23:21,237 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:21,239 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:21,239 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/C is initiating minor compaction (all files) 2024-11-10T17:23:21,239 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/C in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:21,239 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/91d5849b3c9c47f1b27602adcd3af549, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/71c4427093c04c139b40688c29788b55, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/467f52b6c8b641daa6d838b401bff602, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/70ee68fb697d4c15a7c724a35030fe2e] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=48.1 K 2024-11-10T17:23:21,240 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d5849b3c9c47f1b27602adcd3af549, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259396162 2024-11-10T17:23:21,240 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c4427093c04c139b40688c29788b55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731259397452 2024-11-10T17:23:21,241 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 467f52b6c8b641daa6d838b401bff602, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731259397475 2024-11-10T17:23:21,242 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70ee68fb697d4c15a7c724a35030fe2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731259399661 2024-11-10T17:23:21,267 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#C#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:21,268 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/d301aa6417c94a43bd79ab50f9c00ccb is 50, key is test_row_0/C:col10/1731259400303/Put/seqid=0 2024-11-10T17:23:21,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742148_1324 (size=12915) 2024-11-10T17:23:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:21,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:21,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T17:23:21,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/c488a0540d674419884c004d8ceafcf7 is 50, key is test_row_0/A:col10/1731259401473/Put/seqid=0 2024-11-10T17:23:21,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742149_1325 (size=12301) 2024-11-10T17:23:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259461483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259461487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259461488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259461489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259461489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T17:23:21,552 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:21,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T17:23:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T17:23:21,554 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:21,555 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:21,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259461592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259461601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259461601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259461601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259461601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,625 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/3325d21cccdf4b1687f74d44498e7588 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/3325d21cccdf4b1687f74d44498e7588 2024-11-10T17:23:21,630 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/B of 84febc6fa98bac2754e6c1b07a211911 into 3325d21cccdf4b1687f74d44498e7588(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:21,630 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:21,630 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/B, priority=12, startTime=1731259401178; duration=0sec 2024-11-10T17:23:21,630 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:21,630 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:B 2024-11-10T17:23:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T17:23:21,683 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/d301aa6417c94a43bd79ab50f9c00ccb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/d301aa6417c94a43bd79ab50f9c00ccb 2024-11-10T17:23:21,686 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/C of 84febc6fa98bac2754e6c1b07a211911 into d301aa6417c94a43bd79ab50f9c00ccb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:21,687 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:21,687 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/C, priority=12, startTime=1731259401178; duration=0sec 2024-11-10T17:23:21,687 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:21,687 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:C 2024-11-10T17:23:21,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:21,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:21,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259461801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259461811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259461811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259461811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259461812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:21,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:21,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T17:23:21,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/c488a0540d674419884c004d8ceafcf7 2024-11-10T17:23:21,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/d1bfe482d6b84c1898c822e463f23674 is 50, key is test_row_0/B:col10/1731259401473/Put/seqid=0 2024-11-10T17:23:21,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742150_1326 (size=12301) 2024-11-10T17:23:22,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:22,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:22,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259462111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259462121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259462121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259462121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259462121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T17:23:22,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/d1bfe482d6b84c1898c822e463f23674 2024-11-10T17:23:22,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/89f02fe9e4de46a38261fbe9a7b45c79 is 50, key is test_row_0/C:col10/1731259401473/Put/seqid=0 2024-11-10T17:23:22,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742151_1327 (size=12301) 2024-11-10T17:23:22,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259462621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259462631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259462632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259462632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259462632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T17:23:22,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/89f02fe9e4de46a38261fbe9a7b45c79 2024-11-10T17:23:22,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/c488a0540d674419884c004d8ceafcf7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c488a0540d674419884c004d8ceafcf7 2024-11-10T17:23:22,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c488a0540d674419884c004d8ceafcf7, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T17:23:22,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/d1bfe482d6b84c1898c822e463f23674 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d1bfe482d6b84c1898c822e463f23674 2024-11-10T17:23:22,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d1bfe482d6b84c1898c822e463f23674, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T17:23:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/89f02fe9e4de46a38261fbe9a7b45c79 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/89f02fe9e4de46a38261fbe9a7b45c79 2024-11-10T17:23:22,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/89f02fe9e4de46a38261fbe9a7b45c79, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T17:23:22,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 84febc6fa98bac2754e6c1b07a211911 in 1251ms, sequenceid=285, compaction requested=false 2024-11-10T17:23:22,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:22,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T17:23:22,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:22,777 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T17:23:22,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/3a798f6361954fd5a6effdd4e0c5b9f7 is 50, key is test_row_0/A:col10/1731259401488/Put/seqid=0 2024-11-10T17:23:22,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742152_1328 (size=12301) 2024-11-10T17:23:23,185 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/3a798f6361954fd5a6effdd4e0c5b9f7 2024-11-10T17:23:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/c27b093ec46442739b08eb7913cf7f74 is 50, key is test_row_0/B:col10/1731259401488/Put/seqid=0 2024-11-10T17:23:23,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742153_1329 (size=12301) 2024-11-10T17:23:23,599 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/c27b093ec46442739b08eb7913cf7f74 2024-11-10T17:23:23,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/ca5a812dc0f14b439a4afe09c73e08c1 is 50, key is test_row_0/C:col10/1731259401488/Put/seqid=0 2024-11-10T17:23:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:23,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742154_1330 (size=12301) 2024-11-10T17:23:23,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:23,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259463662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259463663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259463665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259463666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259463667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T17:23:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259463771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259463772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259463782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259463782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259463782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259463981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259463982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259463992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259463992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259463992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,035 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/ca5a812dc0f14b439a4afe09c73e08c1 2024-11-10T17:23:24,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/3a798f6361954fd5a6effdd4e0c5b9f7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/3a798f6361954fd5a6effdd4e0c5b9f7 2024-11-10T17:23:24,043 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/3a798f6361954fd5a6effdd4e0c5b9f7, entries=150, sequenceid=302, filesize=12.0 K 2024-11-10T17:23:24,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/c27b093ec46442739b08eb7913cf7f74 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c27b093ec46442739b08eb7913cf7f74 2024-11-10T17:23:24,048 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c27b093ec46442739b08eb7913cf7f74, entries=150, sequenceid=302, filesize=12.0 K 2024-11-10T17:23:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/ca5a812dc0f14b439a4afe09c73e08c1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/ca5a812dc0f14b439a4afe09c73e08c1 2024-11-10T17:23:24,052 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/ca5a812dc0f14b439a4afe09c73e08c1, entries=150, sequenceid=302, filesize=12.0 K 2024-11-10T17:23:24,053 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 84febc6fa98bac2754e6c1b07a211911 in 1277ms, sequenceid=302, compaction requested=true 2024-11-10T17:23:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T17:23:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T17:23:24,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T17:23:24,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4990 sec 2024-11-10T17:23:24,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.5050 sec 2024-11-10T17:23:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:24,296 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:24,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T17:23:24,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/055dfa33bfde4c3bb88c36d377cf6c24 is 50, key is test_row_0/A:col10/1731259404295/Put/seqid=0 2024-11-10T17:23:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742155_1331 (size=17181) 2024-11-10T17:23:24,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/055dfa33bfde4c3bb88c36d377cf6c24 2024-11-10T17:23:24,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/ca0f99d11c014b25862760b3da0102d1 is 50, key is test_row_0/B:col10/1731259404295/Put/seqid=0 2024-11-10T17:23:24,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742156_1332 (size=12301) 2024-11-10T17:23:24,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259464335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259464335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259464336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259464340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259464341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259464451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259464451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259464451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259464451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259464451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259464661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259464661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259464661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259464662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259464662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/ca0f99d11c014b25862760b3da0102d1 2024-11-10T17:23:24,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/fa6375d172244e158418d4e7f90fa33b is 50, key is test_row_0/C:col10/1731259404295/Put/seqid=0 2024-11-10T17:23:24,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742157_1333 (size=12301) 2024-11-10T17:23:24,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259464972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259464972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259464972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259464972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259464972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/fa6375d172244e158418d4e7f90fa33b 2024-11-10T17:23:25,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/055dfa33bfde4c3bb88c36d377cf6c24 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/055dfa33bfde4c3bb88c36d377cf6c24 2024-11-10T17:23:25,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/055dfa33bfde4c3bb88c36d377cf6c24, entries=250, sequenceid=326, filesize=16.8 K 2024-11-10T17:23:25,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/ca0f99d11c014b25862760b3da0102d1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/ca0f99d11c014b25862760b3da0102d1 2024-11-10T17:23:25,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/ca0f99d11c014b25862760b3da0102d1, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T17:23:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/fa6375d172244e158418d4e7f90fa33b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/fa6375d172244e158418d4e7f90fa33b 2024-11-10T17:23:25,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/fa6375d172244e158418d4e7f90fa33b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T17:23:25,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 84febc6fa98bac2754e6c1b07a211911 in 850ms, sequenceid=326, compaction requested=true 2024-11-10T17:23:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:25,146 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 84febc6fa98bac2754e6c1b07a211911:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:25,146 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:25,147 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:25,147 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/B is initiating minor compaction (all files) 2024-11-10T17:23:25,148 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/B in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:25,148 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/3325d21cccdf4b1687f74d44498e7588, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d1bfe482d6b84c1898c822e463f23674, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c27b093ec46442739b08eb7913cf7f74, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/ca0f99d11c014b25862760b3da0102d1] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=48.7 K 2024-11-10T17:23:25,148 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:25,148 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/A is initiating minor compaction (all files) 2024-11-10T17:23:25,148 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/A in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:25,148 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/23c4a0b0cff9462cb02409bb9a9326d1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c488a0540d674419884c004d8ceafcf7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/3a798f6361954fd5a6effdd4e0c5b9f7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/055dfa33bfde4c3bb88c36d377cf6c24] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=53.4 K 2024-11-10T17:23:25,148 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3325d21cccdf4b1687f74d44498e7588, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731259399661 2024-11-10T17:23:25,148 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c4a0b0cff9462cb02409bb9a9326d1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731259399661 2024-11-10T17:23:25,148 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d1bfe482d6b84c1898c822e463f23674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731259400328 2024-11-10T17:23:25,149 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting c488a0540d674419884c004d8ceafcf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731259400328 2024-11-10T17:23:25,149 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c27b093ec46442739b08eb7913cf7f74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731259401487 2024-11-10T17:23:25,149 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a798f6361954fd5a6effdd4e0c5b9f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731259401487 2024-11-10T17:23:25,149 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0f99d11c014b25862760b3da0102d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731259403665 2024-11-10T17:23:25,150 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055dfa33bfde4c3bb88c36d377cf6c24, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731259403661 2024-11-10T17:23:25,159 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:25,159 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:25,159 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f33f023886274e7fb89dca10da4f6d03 is 50, key is test_row_0/B:col10/1731259404295/Put/seqid=0 2024-11-10T17:23:25,160 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/51732922a3764af584e7e5988f63973a is 50, key is test_row_0/A:col10/1731259404295/Put/seqid=0 2024-11-10T17:23:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742158_1334 (size=13051) 2024-11-10T17:23:25,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742159_1335 (size=13051) 2024-11-10T17:23:25,169 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/f33f023886274e7fb89dca10da4f6d03 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f33f023886274e7fb89dca10da4f6d03 2024-11-10T17:23:25,174 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/B of 84febc6fa98bac2754e6c1b07a211911 into f33f023886274e7fb89dca10da4f6d03(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:25,174 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:25,174 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/B, priority=12, startTime=1731259405146; duration=0sec 2024-11-10T17:23:25,174 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:25,174 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:B 2024-11-10T17:23:25,174 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:25,176 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:25,176 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 84febc6fa98bac2754e6c1b07a211911/C is initiating minor compaction (all files) 2024-11-10T17:23:25,176 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 84febc6fa98bac2754e6c1b07a211911/C in TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:25,176 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/d301aa6417c94a43bd79ab50f9c00ccb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/89f02fe9e4de46a38261fbe9a7b45c79, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/ca5a812dc0f14b439a4afe09c73e08c1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/fa6375d172244e158418d4e7f90fa33b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp, totalSize=48.7 K 2024-11-10T17:23:25,177 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d301aa6417c94a43bd79ab50f9c00ccb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731259399661 2024-11-10T17:23:25,177 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f02fe9e4de46a38261fbe9a7b45c79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731259400328 2024-11-10T17:23:25,177 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5a812dc0f14b439a4afe09c73e08c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731259401487 2024-11-10T17:23:25,177 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6375d172244e158418d4e7f90fa33b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731259403665 2024-11-10T17:23:25,185 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 84febc6fa98bac2754e6c1b07a211911#C#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:25,186 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/3acc39a1810c497fa5a04e25e29934bf is 50, key is test_row_0/C:col10/1731259404295/Put/seqid=0 2024-11-10T17:23:25,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742160_1336 (size=13051) 2024-11-10T17:23:25,196 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/3acc39a1810c497fa5a04e25e29934bf as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3acc39a1810c497fa5a04e25e29934bf 2024-11-10T17:23:25,201 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/C of 84febc6fa98bac2754e6c1b07a211911 into 3acc39a1810c497fa5a04e25e29934bf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:25,201 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:25,201 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/C, priority=12, startTime=1731259405146; duration=0sec 2024-11-10T17:23:25,201 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:25,201 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:C 2024-11-10T17:23:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:25,484 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:25,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:23:25,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/af741813493e42578cbbc5832dae896b is 50, key is test_row_0/A:col10/1731259405484/Put/seqid=0 2024-11-10T17:23:25,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742161_1337 (size=12301) 2024-11-10T17:23:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259465533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259465534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259465535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259465535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259465540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,571 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/51732922a3764af584e7e5988f63973a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/51732922a3764af584e7e5988f63973a 2024-11-10T17:23:25,575 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 84febc6fa98bac2754e6c1b07a211911/A of 84febc6fa98bac2754e6c1b07a211911 into 51732922a3764af584e7e5988f63973a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:25,575 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:25,575 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., storeName=84febc6fa98bac2754e6c1b07a211911/A, priority=12, startTime=1731259405146; duration=0sec 2024-11-10T17:23:25,575 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:25,575 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 84febc6fa98bac2754e6c1b07a211911:A 2024-11-10T17:23:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259465652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259465652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259465652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259465652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259465652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T17:23:25,722 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:25,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T17:23:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T17:23:25,725 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:25,725 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:25,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:25,790 INFO [Thread-1245 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1245. 2024-11-10T17:23:25,790 DEBUG [Thread-1245 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:25,790 DEBUG [Thread-1245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,790 DEBUG [Thread-1245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:25,790 INFO [Thread-1243 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1243. 2024-11-10T17:23:25,791 DEBUG [Thread-1243 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:25,791 DEBUG [Thread-1243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,791 DEBUG [Thread-1243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:25,792 INFO [Thread-1239 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1239. 2024-11-10T17:23:25,792 DEBUG [Thread-1239 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:25,792 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,792 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:25,794 INFO [Thread-1241 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1241. 2024-11-10T17:23:25,794 DEBUG [Thread-1241 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:25,794 DEBUG [Thread-1241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,794 DEBUG [Thread-1241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:25,794 INFO [Thread-1237 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1237. 2024-11-10T17:23:25,794 DEBUG [Thread-1237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:25,794 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,794 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:25,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T17:23:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259465861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259465861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259465861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259465861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259465861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:25,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T17:23:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:25,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:25,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/af741813493e42578cbbc5832dae896b 2024-11-10T17:23:25,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/33c1472f699449b3be24b05643e9183d is 50, key is test_row_0/B:col10/1731259405484/Put/seqid=0 2024-11-10T17:23:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742162_1338 (size=12301) 2024-11-10T17:23:26,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T17:23:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T17:23:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259466172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259466172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259466172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259466172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259466172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T17:23:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/33c1472f699449b3be24b05643e9183d 2024-11-10T17:23:26,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/6ef3491d2c09448e889bd4a7eec30d05 is 50, key is test_row_0/C:col10/1731259405484/Put/seqid=0 2024-11-10T17:23:26,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742163_1339 (size=12301) 2024-11-10T17:23:26,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T17:23:26,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T17:23:26,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T17:23:26,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T17:23:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:26,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50206 deadline: 1731259466682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50238 deadline: 1731259466682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50246 deadline: 1731259466682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50244 deadline: 1731259466682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50212 deadline: 1731259466683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:23:26,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/6ef3491d2c09448e889bd4a7eec30d05 2024-11-10T17:23:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/af741813493e42578cbbc5832dae896b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/af741813493e42578cbbc5832dae896b 2024-11-10T17:23:26,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/af741813493e42578cbbc5832dae896b, entries=150, sequenceid=342, filesize=12.0 K 2024-11-10T17:23:26,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/33c1472f699449b3be24b05643e9183d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/33c1472f699449b3be24b05643e9183d 2024-11-10T17:23:26,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/33c1472f699449b3be24b05643e9183d, entries=150, sequenceid=342, filesize=12.0 K 2024-11-10T17:23:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/6ef3491d2c09448e889bd4a7eec30d05 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/6ef3491d2c09448e889bd4a7eec30d05 2024-11-10T17:23:26,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/6ef3491d2c09448e889bd4a7eec30d05, entries=150, sequenceid=342, filesize=12.0 K 2024-11-10T17:23:26,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 84febc6fa98bac2754e6c1b07a211911 in 1254ms, sequenceid=342, compaction requested=false 2024-11-10T17:23:26,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:26,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T17:23:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:26,796 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T17:23:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d056cb6eaa9240ec95765811f9e7c3d7 is 50, key is test_row_0/A:col10/1731259405531/Put/seqid=0 2024-11-10T17:23:26,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742164_1340 (size=12301) 2024-11-10T17:23:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T17:23:27,205 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d056cb6eaa9240ec95765811f9e7c3d7 2024-11-10T17:23:27,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/0c3c0195afc44615a5844d3b94efce88 is 50, key is test_row_0/B:col10/1731259405531/Put/seqid=0 2024-11-10T17:23:27,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742165_1341 (size=12301) 2024-11-10T17:23:27,617 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/0c3c0195afc44615a5844d3b94efce88 2024-11-10T17:23:27,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/f10dd63b77094994807e5b116c16eb9a is 50, key is test_row_0/C:col10/1731259405531/Put/seqid=0 2024-11-10T17:23:27,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742166_1342 (size=12301) 2024-11-10T17:23:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:27,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. as already flushing 2024-11-10T17:23:27,694 INFO [Thread-1234 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1234. 2024-11-10T17:23:27,694 DEBUG [Thread-1234 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:27,694 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,694 INFO [Thread-1226 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1226. 2024-11-10T17:23:27,694 INFO [Thread-1230 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1230. 2024-11-10T17:23:27,694 DEBUG [Thread-1226 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:27,694 DEBUG [Thread-1230 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:27,694 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,694 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,694 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:27,694 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,694 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:27,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:27,696 INFO [Thread-1232 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1232. 2024-11-10T17:23:27,696 DEBUG [Thread-1232 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:27,696 DEBUG [Thread-1232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,696 DEBUG [Thread-1232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:27,702 INFO [Thread-1228 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1228. 2024-11-10T17:23:27,702 DEBUG [Thread-1228 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:27,702 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,702 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:27,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T17:23:28,027 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/f10dd63b77094994807e5b116c16eb9a 2024-11-10T17:23:28,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/d056cb6eaa9240ec95765811f9e7c3d7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d056cb6eaa9240ec95765811f9e7c3d7 2024-11-10T17:23:28,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d056cb6eaa9240ec95765811f9e7c3d7, entries=150, sequenceid=365, filesize=12.0 K 2024-11-10T17:23:28,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/0c3c0195afc44615a5844d3b94efce88 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/0c3c0195afc44615a5844d3b94efce88 2024-11-10T17:23:28,038 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/0c3c0195afc44615a5844d3b94efce88, entries=150, sequenceid=365, filesize=12.0 K 2024-11-10T17:23:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/f10dd63b77094994807e5b116c16eb9a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f10dd63b77094994807e5b116c16eb9a 2024-11-10T17:23:28,041 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f10dd63b77094994807e5b116c16eb9a, entries=150, sequenceid=365, filesize=12.0 K 2024-11-10T17:23:28,042 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=33.54 KB/34350 for 84febc6fa98bac2754e6c1b07a211911 in 1245ms, sequenceid=365, compaction requested=true 2024-11-10T17:23:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 84febc6fa98bac2754e6c1b07a211911: 2024-11-10T17:23:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T17:23:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T17:23:28,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T17:23:28,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-11-10T17:23:28,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.3220 sec 2024-11-10T17:23:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T17:23:29,901 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:29,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T17:23:29,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-10T17:23:29,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-10T17:23:29,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T17:23:29,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-10T17:23:29,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-10T17:23:29,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T17:23:29,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3698 2024-11-10T17:23:29,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3921 2024-11-10T17:23:29,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3650 2024-11-10T17:23:29,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3692 2024-11-10T17:23:29,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3905 2024-11-10T17:23:29,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T17:23:29,902 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:23:29,902 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:23:29,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:29,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:29,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:29,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T17:23:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T17:23:29,906 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259409906"}]},"ts":"1731259409906"} 2024-11-10T17:23:29,907 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T17:23:29,907 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T17:23:29,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:23:29,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, UNASSIGN}] 2024-11-10T17:23:29,909 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, UNASSIGN 2024-11-10T17:23:29,910 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=84febc6fa98bac2754e6c1b07a211911, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:29,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, UNASSIGN because future has completed 2024-11-10T17:23:29,911 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:23:29,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:23:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T17:23:30,063 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:30,063 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:23:30,063 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing 84febc6fa98bac2754e6c1b07a211911, disabling compactions & flushes 2024-11-10T17:23:30,063 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:30,063 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:30,063 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. after waiting 0 ms 2024-11-10T17:23:30,063 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:30,064 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(2902): Flushing 84febc6fa98bac2754e6c1b07a211911 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T17:23:30,067 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/ab2dbf6f7b38451fa2e44df305b0ca89 is 50, key is test_row_0/A:col10/1731259407701/Put/seqid=0 2024-11-10T17:23:30,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742167_1343 (size=12301) 2024-11-10T17:23:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T17:23:30,471 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/ab2dbf6f7b38451fa2e44df305b0ca89 2024-11-10T17:23:30,477 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/4d6caeb8e07d42c2a73f831b7abfaa99 is 50, key is test_row_0/B:col10/1731259407701/Put/seqid=0 2024-11-10T17:23:30,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742168_1344 (size=12301) 2024-11-10T17:23:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T17:23:30,880 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/4d6caeb8e07d42c2a73f831b7abfaa99 2024-11-10T17:23:30,886 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/2cad194d0de444beaeb2aae56a9a3e3f is 50, key is test_row_0/C:col10/1731259407701/Put/seqid=0 2024-11-10T17:23:30,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742169_1345 (size=12301) 2024-11-10T17:23:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T17:23:31,290 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/2cad194d0de444beaeb2aae56a9a3e3f 2024-11-10T17:23:31,294 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/A/ab2dbf6f7b38451fa2e44df305b0ca89 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/ab2dbf6f7b38451fa2e44df305b0ca89 2024-11-10T17:23:31,297 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/ab2dbf6f7b38451fa2e44df305b0ca89, entries=150, sequenceid=373, filesize=12.0 K 2024-11-10T17:23:31,298 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/B/4d6caeb8e07d42c2a73f831b7abfaa99 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4d6caeb8e07d42c2a73f831b7abfaa99 2024-11-10T17:23:31,301 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4d6caeb8e07d42c2a73f831b7abfaa99, entries=150, sequenceid=373, filesize=12.0 K 2024-11-10T17:23:31,302 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/.tmp/C/2cad194d0de444beaeb2aae56a9a3e3f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/2cad194d0de444beaeb2aae56a9a3e3f 2024-11-10T17:23:31,305 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/2cad194d0de444beaeb2aae56a9a3e3f, entries=150, sequenceid=373, filesize=12.0 K 2024-11-10T17:23:31,305 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 84febc6fa98bac2754e6c1b07a211911 in 1242ms, sequenceid=373, compaction requested=true 2024-11-10T17:23:31,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c7f59cf4e5b645269991120d631b529a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/801c72ebf2a44cd09fad84f91cbc6cb2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5fad0141cd3f46858025f0c9041de9be, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/545a125df40c47bdae7b7f9af7634f11, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5b3486c8eec64c3ca56f0fa8cecb8c0f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b9eecdb8616a442fb12cbe001055b3fd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/35ebbc266c0744dbbbbff1d3a333e2e7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/a526cb85d3314181b0ef02ac8a0c42f6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/baf863462ef046d19292dd4de65cadcb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b7003af503d14bf0a7e9e28ff3abc33a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/fc9383bea90744dda27809e9367f9a5b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/7b8b6c6742474c81aaa13ecaf920aad1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/389398a85f6c4fa4b7db390c7a98b7ea, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5ce2bc44eea04b6ba3a141c01d0dae52, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d55aff33795d48839af66b5ae183c3dc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/1bb476df631848368aaf8d2367577359, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d913771841214d48a394577d177e7c66, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/02f648e57556493196aa501da26465b5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/23c4a0b0cff9462cb02409bb9a9326d1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c488a0540d674419884c004d8ceafcf7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/3a798f6361954fd5a6effdd4e0c5b9f7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/055dfa33bfde4c3bb88c36d377cf6c24] to archive 2024-11-10T17:23:31,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:23:31,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c7f59cf4e5b645269991120d631b529a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c7f59cf4e5b645269991120d631b529a 2024-11-10T17:23:31,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/801c72ebf2a44cd09fad84f91cbc6cb2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/801c72ebf2a44cd09fad84f91cbc6cb2 2024-11-10T17:23:31,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5fad0141cd3f46858025f0c9041de9be to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5fad0141cd3f46858025f0c9041de9be 2024-11-10T17:23:31,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/545a125df40c47bdae7b7f9af7634f11 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/545a125df40c47bdae7b7f9af7634f11 2024-11-10T17:23:31,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5b3486c8eec64c3ca56f0fa8cecb8c0f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5b3486c8eec64c3ca56f0fa8cecb8c0f 2024-11-10T17:23:31,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b9eecdb8616a442fb12cbe001055b3fd to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b9eecdb8616a442fb12cbe001055b3fd 2024-11-10T17:23:31,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/35ebbc266c0744dbbbbff1d3a333e2e7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/35ebbc266c0744dbbbbff1d3a333e2e7 2024-11-10T17:23:31,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/a526cb85d3314181b0ef02ac8a0c42f6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/a526cb85d3314181b0ef02ac8a0c42f6 2024-11-10T17:23:31,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/baf863462ef046d19292dd4de65cadcb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/baf863462ef046d19292dd4de65cadcb 2024-11-10T17:23:31,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b7003af503d14bf0a7e9e28ff3abc33a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/b7003af503d14bf0a7e9e28ff3abc33a 2024-11-10T17:23:31,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/fc9383bea90744dda27809e9367f9a5b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/fc9383bea90744dda27809e9367f9a5b 2024-11-10T17:23:31,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/7b8b6c6742474c81aaa13ecaf920aad1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/7b8b6c6742474c81aaa13ecaf920aad1 2024-11-10T17:23:31,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/389398a85f6c4fa4b7db390c7a98b7ea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/389398a85f6c4fa4b7db390c7a98b7ea 2024-11-10T17:23:31,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5ce2bc44eea04b6ba3a141c01d0dae52 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/5ce2bc44eea04b6ba3a141c01d0dae52 2024-11-10T17:23:31,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d55aff33795d48839af66b5ae183c3dc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d55aff33795d48839af66b5ae183c3dc 2024-11-10T17:23:31,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/1bb476df631848368aaf8d2367577359 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/1bb476df631848368aaf8d2367577359 2024-11-10T17:23:31,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d913771841214d48a394577d177e7c66 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d913771841214d48a394577d177e7c66 2024-11-10T17:23:31,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/02f648e57556493196aa501da26465b5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/02f648e57556493196aa501da26465b5 2024-11-10T17:23:31,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/23c4a0b0cff9462cb02409bb9a9326d1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/23c4a0b0cff9462cb02409bb9a9326d1 2024-11-10T17:23:31,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c488a0540d674419884c004d8ceafcf7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/c488a0540d674419884c004d8ceafcf7 2024-11-10T17:23:31,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/3a798f6361954fd5a6effdd4e0c5b9f7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/3a798f6361954fd5a6effdd4e0c5b9f7 2024-11-10T17:23:31,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/055dfa33bfde4c3bb88c36d377cf6c24 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/055dfa33bfde4c3bb88c36d377cf6c24 2024-11-10T17:23:31,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/aa20ada6338749e1b2e9dd3f25e6d45f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c54cdc3116404d288d308ccbdad57f8e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6618a3c9773d4de0a134de1f793ad775, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b33b64a3672043bd89c37389e326d631, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b1c412c85e2e43608030e970c4984e8f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f904b13b79bb427886eca1dc7cbaf918, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/af18ab68d3ee47db8259895f44ec2acb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6154220da2e64be79ed4175998e9d73b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/77d748e3c68b4d2fbfb328d8a6f49e3a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/32af7323ab8649ae9f6471efefb3d53c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f26d6fcb26bc4e5ebd178605981b55cf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d985e9d0e77a44689b90c1dddfa64b27, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6ff66f143e5249b1889237020c7059e3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b10e5325a6d441e5923a86b73147349b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/963cf6de63c448d5a32c54e7234bf722, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4143278ad83d4e62b92bfe59975a74cf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/3325d21cccdf4b1687f74d44498e7588, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/23fdaf0ce6824cf89ed0cc1f2919694e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d1bfe482d6b84c1898c822e463f23674, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c27b093ec46442739b08eb7913cf7f74, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/ca0f99d11c014b25862760b3da0102d1] to archive 2024-11-10T17:23:31,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:23:31,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/aa20ada6338749e1b2e9dd3f25e6d45f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/aa20ada6338749e1b2e9dd3f25e6d45f 2024-11-10T17:23:31,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c54cdc3116404d288d308ccbdad57f8e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c54cdc3116404d288d308ccbdad57f8e 2024-11-10T17:23:31,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6618a3c9773d4de0a134de1f793ad775 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6618a3c9773d4de0a134de1f793ad775 2024-11-10T17:23:31,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b33b64a3672043bd89c37389e326d631 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b33b64a3672043bd89c37389e326d631 2024-11-10T17:23:31,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b1c412c85e2e43608030e970c4984e8f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b1c412c85e2e43608030e970c4984e8f 2024-11-10T17:23:31,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f904b13b79bb427886eca1dc7cbaf918 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f904b13b79bb427886eca1dc7cbaf918 2024-11-10T17:23:31,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/af18ab68d3ee47db8259895f44ec2acb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/af18ab68d3ee47db8259895f44ec2acb 2024-11-10T17:23:31,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6154220da2e64be79ed4175998e9d73b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6154220da2e64be79ed4175998e9d73b 2024-11-10T17:23:31,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/96b1e2c6b2fb4d7abf4c05d6ff96c37a 2024-11-10T17:23:31,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/77d748e3c68b4d2fbfb328d8a6f49e3a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/77d748e3c68b4d2fbfb328d8a6f49e3a 2024-11-10T17:23:31,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/32af7323ab8649ae9f6471efefb3d53c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/32af7323ab8649ae9f6471efefb3d53c 2024-11-10T17:23:31,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f26d6fcb26bc4e5ebd178605981b55cf to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f26d6fcb26bc4e5ebd178605981b55cf 2024-11-10T17:23:31,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d985e9d0e77a44689b90c1dddfa64b27 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d985e9d0e77a44689b90c1dddfa64b27 2024-11-10T17:23:31,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6ff66f143e5249b1889237020c7059e3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/6ff66f143e5249b1889237020c7059e3 2024-11-10T17:23:31,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b10e5325a6d441e5923a86b73147349b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/b10e5325a6d441e5923a86b73147349b 2024-11-10T17:23:31,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/963cf6de63c448d5a32c54e7234bf722 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/963cf6de63c448d5a32c54e7234bf722 2024-11-10T17:23:31,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4143278ad83d4e62b92bfe59975a74cf to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4143278ad83d4e62b92bfe59975a74cf 2024-11-10T17:23:31,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/3325d21cccdf4b1687f74d44498e7588 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/3325d21cccdf4b1687f74d44498e7588 2024-11-10T17:23:31,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/23fdaf0ce6824cf89ed0cc1f2919694e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/23fdaf0ce6824cf89ed0cc1f2919694e 2024-11-10T17:23:31,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d1bfe482d6b84c1898c822e463f23674 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/d1bfe482d6b84c1898c822e463f23674 2024-11-10T17:23:31,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c27b093ec46442739b08eb7913cf7f74 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/c27b093ec46442739b08eb7913cf7f74 2024-11-10T17:23:31,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/ca0f99d11c014b25862760b3da0102d1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/ca0f99d11c014b25862760b3da0102d1 2024-11-10T17:23:31,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/478cfb48fb1e44f8a312a2e75088f96b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/34fd5f9870d947789c8c1e7a40a66c20, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f65cf2575b5740bb8087e9a5f4558799, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3600f078661542e5a6cd8d3a8eeb1321, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/365310dd0c274d0db7a757380d21f2ae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/68853f1d737547daad0265318281b402, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/67b63b3ddec94bf69005c9bac048b8eb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/5c687ed9b2f249fdbc4a3e3ec1106ec0, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e78146041e1947f4b26ed595f9d8083d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/908d8ba695554947b19ec52881ee584e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e64198cc21d64a72acbd5f0190fa1e56, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/7ffe61e7c5b94f88b9467bb85305c112, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/91d5849b3c9c47f1b27602adcd3af549, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/999b36003ac84d30b213670b306675a5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/71c4427093c04c139b40688c29788b55, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/467f52b6c8b641daa6d838b401bff602, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/d301aa6417c94a43bd79ab50f9c00ccb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/70ee68fb697d4c15a7c724a35030fe2e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/89f02fe9e4de46a38261fbe9a7b45c79, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/ca5a812dc0f14b439a4afe09c73e08c1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/fa6375d172244e158418d4e7f90fa33b] to archive 2024-11-10T17:23:31,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:23:31,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/478cfb48fb1e44f8a312a2e75088f96b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/478cfb48fb1e44f8a312a2e75088f96b 2024-11-10T17:23:31,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/34fd5f9870d947789c8c1e7a40a66c20 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/34fd5f9870d947789c8c1e7a40a66c20 2024-11-10T17:23:31,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f65cf2575b5740bb8087e9a5f4558799 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f65cf2575b5740bb8087e9a5f4558799 2024-11-10T17:23:31,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3600f078661542e5a6cd8d3a8eeb1321 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3600f078661542e5a6cd8d3a8eeb1321 2024-11-10T17:23:31,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/365310dd0c274d0db7a757380d21f2ae to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/365310dd0c274d0db7a757380d21f2ae 2024-11-10T17:23:31,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/68853f1d737547daad0265318281b402 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/68853f1d737547daad0265318281b402 2024-11-10T17:23:31,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/67b63b3ddec94bf69005c9bac048b8eb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/67b63b3ddec94bf69005c9bac048b8eb 2024-11-10T17:23:31,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/5c687ed9b2f249fdbc4a3e3ec1106ec0 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/5c687ed9b2f249fdbc4a3e3ec1106ec0 2024-11-10T17:23:31,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e78146041e1947f4b26ed595f9d8083d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e78146041e1947f4b26ed595f9d8083d 2024-11-10T17:23:31,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/908d8ba695554947b19ec52881ee584e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/908d8ba695554947b19ec52881ee584e 2024-11-10T17:23:31,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e64198cc21d64a72acbd5f0190fa1e56 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/e64198cc21d64a72acbd5f0190fa1e56 2024-11-10T17:23:31,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/7ffe61e7c5b94f88b9467bb85305c112 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/7ffe61e7c5b94f88b9467bb85305c112 2024-11-10T17:23:31,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/dc9f8fb9a90f4f5fb3ad639bc361f8c8 2024-11-10T17:23:31,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/91d5849b3c9c47f1b27602adcd3af549 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/91d5849b3c9c47f1b27602adcd3af549 2024-11-10T17:23:31,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/999b36003ac84d30b213670b306675a5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/999b36003ac84d30b213670b306675a5 2024-11-10T17:23:31,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/71c4427093c04c139b40688c29788b55 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/71c4427093c04c139b40688c29788b55 2024-11-10T17:23:31,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/467f52b6c8b641daa6d838b401bff602 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/467f52b6c8b641daa6d838b401bff602 2024-11-10T17:23:31,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/d301aa6417c94a43bd79ab50f9c00ccb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/d301aa6417c94a43bd79ab50f9c00ccb 2024-11-10T17:23:31,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/70ee68fb697d4c15a7c724a35030fe2e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/70ee68fb697d4c15a7c724a35030fe2e 2024-11-10T17:23:31,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/89f02fe9e4de46a38261fbe9a7b45c79 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/89f02fe9e4de46a38261fbe9a7b45c79 2024-11-10T17:23:31,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/ca5a812dc0f14b439a4afe09c73e08c1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/ca5a812dc0f14b439a4afe09c73e08c1 2024-11-10T17:23:31,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/fa6375d172244e158418d4e7f90fa33b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/fa6375d172244e158418d4e7f90fa33b 2024-11-10T17:23:31,371 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-10T17:23:31,371 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. 2024-11-10T17:23:31,371 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for 84febc6fa98bac2754e6c1b07a211911: Waiting for close lock at 1731259410063Running coprocessor pre-close hooks at 1731259410063Disabling compacts and flushes for region at 1731259410063Disabling writes for close at 1731259410063Obtaining lock to block concurrent updates at 1731259410064 (+1 ms)Preparing flush snapshotting stores in 84febc6fa98bac2754e6c1b07a211911 at 1731259410064Finished memstore snapshotting TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731259410064Flushing stores of TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911. at 1731259410064Flushing 84febc6fa98bac2754e6c1b07a211911/A: creating writer at 1731259410064Flushing 84febc6fa98bac2754e6c1b07a211911/A: appending metadata at 1731259410067 (+3 ms)Flushing 84febc6fa98bac2754e6c1b07a211911/A: closing flushed file at 1731259410067Flushing 84febc6fa98bac2754e6c1b07a211911/B: creating writer at 1731259410474 (+407 ms)Flushing 84febc6fa98bac2754e6c1b07a211911/B: appending metadata at 1731259410476 (+2 ms)Flushing 84febc6fa98bac2754e6c1b07a211911/B: closing flushed file at 1731259410476Flushing 84febc6fa98bac2754e6c1b07a211911/C: creating writer at 1731259410884 (+408 ms)Flushing 84febc6fa98bac2754e6c1b07a211911/C: appending metadata at 1731259410886 (+2 ms)Flushing 84febc6fa98bac2754e6c1b07a211911/C: closing flushed file at 1731259410886Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49747269: reopening flushed file at 1731259411294 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2342b22d: reopening flushed file at 1731259411297 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f1bc629: reopening flushed file at 1731259411301 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 84febc6fa98bac2754e6c1b07a211911 in 1242ms, sequenceid=373, compaction requested=true at 1731259411305 (+4 ms)Writing region close event to WAL at 1731259411368 (+63 ms)Running coprocessor post-close hooks at 1731259411371 (+3 ms)Closed at 1731259411371 2024-11-10T17:23:31,373 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed 84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:31,373 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=84febc6fa98bac2754e6c1b07a211911, regionState=CLOSED 2024-11-10T17:23:31,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:23:31,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T17:23:31,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseRegionProcedure 84febc6fa98bac2754e6c1b07a211911, server=8e3de9160f9f,36421,1731259326553 in 1.4640 sec 2024-11-10T17:23:31,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-10T17:23:31,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=84febc6fa98bac2754e6c1b07a211911, UNASSIGN in 1.4680 sec 2024-11-10T17:23:31,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T17:23:31,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4710 sec 2024-11-10T17:23:31,382 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259411382"}]},"ts":"1731259411382"} 2024-11-10T17:23:31,383 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T17:23:31,383 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T17:23:31,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-10T17:23:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T17:23:32,052 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:32,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T17:23:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:32,054 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T17:23:32,054 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=76, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:32,057 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:32,059 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/recovered.edits] 2024-11-10T17:23:32,061 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/51732922a3764af584e7e5988f63973a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/51732922a3764af584e7e5988f63973a 2024-11-10T17:23:32,062 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/ab2dbf6f7b38451fa2e44df305b0ca89 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/ab2dbf6f7b38451fa2e44df305b0ca89 2024-11-10T17:23:32,063 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/af741813493e42578cbbc5832dae896b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/af741813493e42578cbbc5832dae896b 2024-11-10T17:23:32,064 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d056cb6eaa9240ec95765811f9e7c3d7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/A/d056cb6eaa9240ec95765811f9e7c3d7 2024-11-10T17:23:32,066 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/0c3c0195afc44615a5844d3b94efce88 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/0c3c0195afc44615a5844d3b94efce88 2024-11-10T17:23:32,066 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/33c1472f699449b3be24b05643e9183d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/33c1472f699449b3be24b05643e9183d 2024-11-10T17:23:32,067 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4d6caeb8e07d42c2a73f831b7abfaa99 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/4d6caeb8e07d42c2a73f831b7abfaa99 2024-11-10T17:23:32,068 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f33f023886274e7fb89dca10da4f6d03 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/B/f33f023886274e7fb89dca10da4f6d03 2024-11-10T17:23:32,070 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/2cad194d0de444beaeb2aae56a9a3e3f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/2cad194d0de444beaeb2aae56a9a3e3f 2024-11-10T17:23:32,071 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3acc39a1810c497fa5a04e25e29934bf to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/3acc39a1810c497fa5a04e25e29934bf 2024-11-10T17:23:32,072 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/6ef3491d2c09448e889bd4a7eec30d05 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/6ef3491d2c09448e889bd4a7eec30d05 2024-11-10T17:23:32,073 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f10dd63b77094994807e5b116c16eb9a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/C/f10dd63b77094994807e5b116c16eb9a 2024-11-10T17:23:32,075 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/recovered.edits/376.seqid to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911/recovered.edits/376.seqid 2024-11-10T17:23:32,076 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/84febc6fa98bac2754e6c1b07a211911 2024-11-10T17:23:32,076 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T17:23:32,079 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=76, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:32,081 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T17:23:32,083 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T17:23:32,084 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=76, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:32,084 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T17:23:32,084 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731259412084"}]},"ts":"9223372036854775807"} 2024-11-10T17:23:32,086 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T17:23:32,086 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 84febc6fa98bac2754e6c1b07a211911, NAME => 'TestAcidGuarantees,,1731259384591.84febc6fa98bac2754e6c1b07a211911.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T17:23:32,086 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T17:23:32,086 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731259412086"}]},"ts":"9223372036854775807"} 2024-11-10T17:23:32,088 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T17:23:32,089 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=76, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:32,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-10T17:23:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T17:23:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T17:23:32,162 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:32,171 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 245), OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 485), ProcessCount=11 (was 11), AvailableMemoryMB=6122 (was 6199) 2024-11-10T17:23:32,179 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=6122 2024-11-10T17:23:32,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:23:32,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:23:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:32,182 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T17:23:32,183 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:32,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 77 2024-11-10T17:23:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-10T17:23:32,183 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T17:23:32,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742170_1346 (size=959) 2024-11-10T17:23:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-10T17:23:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-10T17:23:32,590 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:23:32,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742171_1347 (size=53) 2024-11-10T17:23:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-10T17:23:32,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:23:32,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a43b207a293e67a9738e21c91e1dc09e, disabling compactions & flushes 2024-11-10T17:23:32,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:32,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:32,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. after waiting 0 ms 2024-11-10T17:23:32,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:32,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:32,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a43b207a293e67a9738e21c91e1dc09e: Waiting for close lock at 1731259412995Disabling compacts and flushes for region at 1731259412995Disabling writes for close at 1731259412995Writing region close event to WAL at 1731259412995Closed at 1731259412995 2024-11-10T17:23:32,996 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T17:23:32,996 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731259412996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731259412996"}]},"ts":"1731259412996"} 2024-11-10T17:23:32,998 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T17:23:32,998 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T17:23:32,999 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259412998"}]},"ts":"1731259412998"} 2024-11-10T17:23:33,000 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T17:23:33,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, ASSIGN}] 2024-11-10T17:23:33,001 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, ASSIGN 2024-11-10T17:23:33,001 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, ASSIGN; state=OFFLINE, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=false 2024-11-10T17:23:33,152 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:33,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, ASSIGN because future has completed 2024-11-10T17:23:33,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:23:33,308 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:33,308 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:23:33,308 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,309 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:23:33,309 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,309 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,310 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,311 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a43b207a293e67a9738e21c91e1dc09e columnFamilyName A 2024-11-10T17:23:33,311 DEBUG [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:33,311 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(327): Store=a43b207a293e67a9738e21c91e1dc09e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:33,312 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,312 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a43b207a293e67a9738e21c91e1dc09e columnFamilyName B 2024-11-10T17:23:33,312 DEBUG [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:33,313 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(327): Store=a43b207a293e67a9738e21c91e1dc09e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:33,313 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,314 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a43b207a293e67a9738e21c91e1dc09e columnFamilyName C 2024-11-10T17:23:33,314 DEBUG [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:33,314 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(327): Store=a43b207a293e67a9738e21c91e1dc09e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:33,314 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,315 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,315 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,315 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,315 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,316 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:23:33,317 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,318 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:23:33,319 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened a43b207a293e67a9738e21c91e1dc09e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74411874, jitterRate=0.10882332921028137}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:23:33,319 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:33,319 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for a43b207a293e67a9738e21c91e1dc09e: Running coprocessor pre-open hook at 1731259413309Writing region info on filesystem at 1731259413309Initializing all the Stores at 1731259413309Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259413309Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259413310 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259413310Cleaning up temporary data from old regions at 1731259413315 (+5 ms)Running coprocessor post-open hooks at 1731259413319 (+4 ms)Region opened successfully at 1731259413319 2024-11-10T17:23:33,320 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., pid=79, masterSystemTime=1731259413305 2024-11-10T17:23:33,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-10T17:23:33,322 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:33,322 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:33,323 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:33,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:23:33,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T17:23:33,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 in 171 msec 2024-11-10T17:23:33,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-10T17:23:33,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, ASSIGN in 327 msec 2024-11-10T17:23:33,329 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T17:23:33,329 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259413329"}]},"ts":"1731259413329"} 2024-11-10T17:23:33,331 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T17:23:33,332 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T17:23:33,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-10T17:23:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-10T17:23:34,332 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:34,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:34,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:34,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:34,335 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680288e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:34,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559577a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:34,337 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:34,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:34,339 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T17:23:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,40007,1731259325831 2024-11-10T17:23:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ada58cc 2024-11-10T17:23:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T17:23:34,341 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T17:23:34,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:23:34,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:23:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T17:23:34,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742172_1348 (size=995) 2024-11-10T17:23:34,753 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T17:23:34,754 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T17:23:34,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:23:34,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, REOPEN/MOVE}] 2024-11-10T17:23:34,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, REOPEN/MOVE 2024-11-10T17:23:34,760 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:34,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, REOPEN/MOVE because future has completed 2024-11-10T17:23:34,762 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:23:34,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:23:34,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T17:23:34,914 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:34,914 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:23:34,914 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing a43b207a293e67a9738e21c91e1dc09e, disabling compactions & flushes 2024-11-10T17:23:34,914 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:34,914 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:34,914 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. after waiting 0 ms 2024-11-10T17:23:34,914 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:34,918 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T17:23:34,918 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:34,918 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for a43b207a293e67a9738e21c91e1dc09e: Waiting for close lock at 1731259414914Running coprocessor pre-close hooks at 1731259414914Disabling compacts and flushes for region at 1731259414914Disabling writes for close at 1731259414914Writing region close event to WAL at 1731259414915 (+1 ms)Running coprocessor post-close hooks at 1731259414918 (+3 ms)Closed at 1731259414918 2024-11-10T17:23:34,918 WARN [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionServer(3299): Not adding moved region record: a43b207a293e67a9738e21c91e1dc09e to self. 2024-11-10T17:23:34,920 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:34,920 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=CLOSED 2024-11-10T17:23:34,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:23:34,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T17:23:34,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 in 160 msec 2024-11-10T17:23:34,924 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, REOPEN/MOVE; state=CLOSED, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=true 2024-11-10T17:23:35,074 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, REOPEN/MOVE because future has completed 2024-11-10T17:23:35,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:23:35,230 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,230 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7752): Opening region: {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:23:35,230 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,230 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:23:35,230 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7794): checking encryption for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,230 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7797): checking classloading for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,231 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,232 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a43b207a293e67a9738e21c91e1dc09e columnFamilyName A 2024-11-10T17:23:35,233 DEBUG [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:35,233 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(327): Store=a43b207a293e67a9738e21c91e1dc09e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:35,234 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,234 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a43b207a293e67a9738e21c91e1dc09e columnFamilyName B 2024-11-10T17:23:35,234 DEBUG [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:35,234 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(327): Store=a43b207a293e67a9738e21c91e1dc09e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:35,234 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,235 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a43b207a293e67a9738e21c91e1dc09e columnFamilyName C 2024-11-10T17:23:35,235 DEBUG [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:35,235 INFO [StoreOpener-a43b207a293e67a9738e21c91e1dc09e-1 {}] regionserver.HStore(327): Store=a43b207a293e67a9738e21c91e1dc09e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:23:35,235 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1038): replaying wal for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,236 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,236 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,237 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1048): stopping wal replay for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,237 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1060): Cleaning up temporary data for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,237 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:23:35,238 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1093): writing seq id for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,239 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1114): Opened a43b207a293e67a9738e21c91e1dc09e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60092646, jitterRate=-0.10454979538917542}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:23:35,239 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,239 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1006): Region open journal for a43b207a293e67a9738e21c91e1dc09e: Running coprocessor pre-open hook at 1731259415230Writing region info on filesystem at 1731259415230Initializing all the Stores at 1731259415231 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259415231Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259415231Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259415231Cleaning up temporary data from old regions at 1731259415237 (+6 ms)Running coprocessor post-open hooks at 1731259415239 (+2 ms)Region opened successfully at 1731259415239 2024-11-10T17:23:35,240 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., pid=84, masterSystemTime=1731259415227 2024-11-10T17:23:35,241 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,241 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,242 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=OPEN, openSeqNum=5, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:23:35,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-11-10T17:23:35,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 in 167 msec 2024-11-10T17:23:35,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-10T17:23:35,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, REOPEN/MOVE in 487 msec 2024-11-10T17:23:35,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T17:23:35,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-10T17:23:35,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-10T17:23:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T17:23:35,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:35,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,252 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed0b7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,254 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ad27ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355e01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d914108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,258 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5a2636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,259 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c17b560, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,261 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b6ecb12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,262 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f29aa99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,264 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df374ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,265 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336b47eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,266 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2681afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,267 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8453ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,269 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@704599f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,270 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45148463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,272 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a3ad38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,273 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391921bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,275 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3834c6fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,276 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7effb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:23:35,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:23:35,277 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:23:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:23:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:23:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b64a97f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:23:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:23:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:35,278 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:23:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ea4b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:23:35,279 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,279 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,279 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,279 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,280 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,280 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,282 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,282 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:35,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,282 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,283 DEBUG [Thread-1579 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,284 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,284 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:23:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-10T17:23:35,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-10T17:23:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:23:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:23:35,286 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:23:35,288 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:35,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:23:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:35,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:23:35,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111057d344464e154c7784cc3c2ebbc69f07_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259415297/Put/seqid=0 2024-11-10T17:23:35,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742173_1349 (size=12154) 2024-11-10T17:23:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259475346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259475349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259475353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259475355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259475357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-10T17:23:35,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:35,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259475461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259475461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259475472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259475472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259475472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-10T17:23:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259475672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259475681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259475691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259475691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259475691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,748 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:35,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111057d344464e154c7784cc3c2ebbc69f07_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111057d344464e154c7784cc3c2ebbc69f07_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:35,752 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5279420d1b9b45cf936eba3da67405dd, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:35,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5279420d1b9b45cf936eba3da67405dd is 175, key is test_row_0/A:col10/1731259415297/Put/seqid=0 2024-11-10T17:23:35,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:35,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:35,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742174_1350 (size=30955) 2024-11-10T17:23:35,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:35,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:35,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:35,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-10T17:23:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259475982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259475991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259476001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259476001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259476001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,160 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5279420d1b9b45cf936eba3da67405dd 2024-11-10T17:23:36,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/b429b7d275524630be7c6925f8655996 is 50, key is test_row_0/B:col10/1731259415297/Put/seqid=0 2024-11-10T17:23:36,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742175_1351 (size=12001) 2024-11-10T17:23:36,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/b429b7d275524630be7c6925f8655996 2024-11-10T17:23:36,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:36,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:36,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a97b6ad5018e4b70a22f2639ff55c167 is 50, key is test_row_0/C:col10/1731259415297/Put/seqid=0 2024-11-10T17:23:36,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742176_1352 (size=12001) 2024-11-10T17:23:36,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-10T17:23:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259476491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259476501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:36,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259476522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259476522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259476522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:36,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a97b6ad5018e4b70a22f2639ff55c167 2024-11-10T17:23:36,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5279420d1b9b45cf936eba3da67405dd as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5279420d1b9b45cf936eba3da67405dd 2024-11-10T17:23:36,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5279420d1b9b45cf936eba3da67405dd, entries=150, sequenceid=16, filesize=30.2 K 2024-11-10T17:23:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/b429b7d275524630be7c6925f8655996 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b429b7d275524630be7c6925f8655996 2024-11-10T17:23:36,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b429b7d275524630be7c6925f8655996, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T17:23:36,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a97b6ad5018e4b70a22f2639ff55c167 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a97b6ad5018e4b70a22f2639ff55c167 2024-11-10T17:23:36,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a97b6ad5018e4b70a22f2639ff55c167, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T17:23:36,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a43b207a293e67a9738e21c91e1dc09e in 1347ms, sequenceid=16, compaction requested=false 2024-11-10T17:23:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T17:23:36,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:36,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-10T17:23:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:36,673 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:23:36,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d6ce60277feb4e71af4db04ed98d852c_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259415341/Put/seqid=0 2024-11-10T17:23:36,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742177_1353 (size=12154) 2024-11-10T17:23:36,899 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T17:23:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:37,096 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d6ce60277feb4e71af4db04ed98d852c_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d6ce60277feb4e71af4db04ed98d852c_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5e9a8894858c41be93fc2082ca16b2bf, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:37,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5e9a8894858c41be93fc2082ca16b2bf is 175, key is test_row_0/A:col10/1731259415341/Put/seqid=0 2024-11-10T17:23:37,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742178_1354 (size=30955) 2024-11-10T17:23:37,115 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5e9a8894858c41be93fc2082ca16b2bf 2024-11-10T17:23:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/f106948845654af5837335c253bf109e is 50, key is test_row_0/B:col10/1731259415341/Put/seqid=0 2024-11-10T17:23:37,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742179_1355 (size=12001) 2024-11-10T17:23:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-10T17:23:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:37,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:37,547 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/f106948845654af5837335c253bf109e 2024-11-10T17:23:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259477542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259477542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259477543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,552 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,553 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/4c700afccd484879ae6ddfd91b2c0402 is 50, key is test_row_0/C:col10/1731259415341/Put/seqid=0 2024-11-10T17:23:37,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742180_1356 (size=12001) 2024-11-10T17:23:37,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259477549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259477552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259477662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259477671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259477671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259477882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259477882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259477882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:37,960 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/4c700afccd484879ae6ddfd91b2c0402 2024-11-10T17:23:37,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5e9a8894858c41be93fc2082ca16b2bf as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5e9a8894858c41be93fc2082ca16b2bf 2024-11-10T17:23:37,968 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5e9a8894858c41be93fc2082ca16b2bf, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T17:23:37,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/f106948845654af5837335c253bf109e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/f106948845654af5837335c253bf109e 2024-11-10T17:23:37,973 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/f106948845654af5837335c253bf109e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T17:23:37,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/4c700afccd484879ae6ddfd91b2c0402 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4c700afccd484879ae6ddfd91b2c0402 2024-11-10T17:23:37,981 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4c700afccd484879ae6ddfd91b2c0402, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T17:23:37,982 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a43b207a293e67a9738e21c91e1dc09e in 1309ms, sequenceid=40, compaction requested=false 2024-11-10T17:23:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-10T17:23:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-10T17:23:37,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-10T17:23:37,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6990 sec 2024-11-10T17:23:37,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.7070 sec 2024-11-10T17:23:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:38,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:38,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:23:38,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103af5d07958f94dfaaf94aedcccaf3f56_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259417547/Put/seqid=0 2024-11-10T17:23:38,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742181_1357 (size=17034) 2024-11-10T17:23:38,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:38,259 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103af5d07958f94dfaaf94aedcccaf3f56_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103af5d07958f94dfaaf94aedcccaf3f56_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:38,261 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/baa447ccc0984ce784729236b383b73f, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:38,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/baa447ccc0984ce784729236b383b73f is 175, key is test_row_0/A:col10/1731259417547/Put/seqid=0 2024-11-10T17:23:38,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742182_1358 (size=48139) 2024-11-10T17:23:38,277 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/baa447ccc0984ce784729236b383b73f 2024-11-10T17:23:38,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259478277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/a52d4b4bd1544e81928144667f7ddab8 is 50, key is test_row_0/B:col10/1731259417547/Put/seqid=0 2024-11-10T17:23:38,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259478284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742183_1359 (size=12001) 2024-11-10T17:23:38,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/a52d4b4bd1544e81928144667f7ddab8 2024-11-10T17:23:38,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259478286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/292d6643cd4f402a97fe6610d23efdba is 50, key is test_row_0/C:col10/1731259417547/Put/seqid=0 2024-11-10T17:23:38,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742184_1360 (size=12001) 2024-11-10T17:23:38,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/292d6643cd4f402a97fe6610d23efdba 2024-11-10T17:23:38,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/baa447ccc0984ce784729236b383b73f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/baa447ccc0984ce784729236b383b73f 2024-11-10T17:23:38,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/baa447ccc0984ce784729236b383b73f, entries=250, sequenceid=55, filesize=47.0 K 2024-11-10T17:23:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/a52d4b4bd1544e81928144667f7ddab8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/a52d4b4bd1544e81928144667f7ddab8 2024-11-10T17:23:38,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/a52d4b4bd1544e81928144667f7ddab8, entries=150, sequenceid=55, filesize=11.7 K 2024-11-10T17:23:38,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/292d6643cd4f402a97fe6610d23efdba as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/292d6643cd4f402a97fe6610d23efdba 2024-11-10T17:23:38,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/292d6643cd4f402a97fe6610d23efdba, entries=150, sequenceid=55, filesize=11.7 K 2024-11-10T17:23:38,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a43b207a293e67a9738e21c91e1dc09e in 150ms, sequenceid=55, compaction requested=true 2024-11-10T17:23:38,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:38,349 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:38,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:38,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:38,350 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:38,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:38,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:38,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:38,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:38,350 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:38,350 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/A is initiating minor compaction (all files) 2024-11-10T17:23:38,351 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/A in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:38,351 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5279420d1b9b45cf936eba3da67405dd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5e9a8894858c41be93fc2082ca16b2bf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/baa447ccc0984ce784729236b383b73f] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=107.5 K 2024-11-10T17:23:38,351 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:38,351 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5279420d1b9b45cf936eba3da67405dd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5e9a8894858c41be93fc2082ca16b2bf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/baa447ccc0984ce784729236b383b73f] 2024-11-10T17:23:38,351 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:38,351 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/B is initiating minor compaction (all files) 2024-11-10T17:23:38,351 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/B in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:38,351 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b429b7d275524630be7c6925f8655996, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/f106948845654af5837335c253bf109e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/a52d4b4bd1544e81928144667f7ddab8] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.2 K 2024-11-10T17:23:38,352 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5279420d1b9b45cf936eba3da67405dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731259415297 2024-11-10T17:23:38,352 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b429b7d275524630be7c6925f8655996, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731259415297 2024-11-10T17:23:38,352 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9a8894858c41be93fc2082ca16b2bf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259415328 2024-11-10T17:23:38,352 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f106948845654af5837335c253bf109e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259415328 2024-11-10T17:23:38,352 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa447ccc0984ce784729236b383b73f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259417547 2024-11-10T17:23:38,353 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting a52d4b4bd1544e81928144667f7ddab8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259417547 2024-11-10T17:23:38,366 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:38,366 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/4066066c71b0498a820321f150c29058 is 50, key is test_row_0/B:col10/1731259417547/Put/seqid=0 2024-11-10T17:23:38,373 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:38,389 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111045f085f09f2c4a139e7ddde2e088d71e_a43b207a293e67a9738e21c91e1dc09e store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:38,391 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111045f085f09f2c4a139e7ddde2e088d71e_a43b207a293e67a9738e21c91e1dc09e, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:38,392 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111045f085f09f2c4a139e7ddde2e088d71e_a43b207a293e67a9738e21c91e1dc09e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:38,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742185_1361 (size=12104) 2024-11-10T17:23:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:38,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:38,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T17:23:38,403 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/4066066c71b0498a820321f150c29058 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/4066066c71b0498a820321f150c29058 2024-11-10T17:23:38,409 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/B of a43b207a293e67a9738e21c91e1dc09e into 4066066c71b0498a820321f150c29058(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:38,409 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:38,410 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/B, priority=13, startTime=1731259418349; duration=0sec 2024-11-10T17:23:38,410 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:38,410 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:B 2024-11-10T17:23:38,410 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:38,413 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:38,413 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/C is initiating minor compaction (all files) 2024-11-10T17:23:38,414 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/C in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:38,414 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a97b6ad5018e4b70a22f2639ff55c167, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4c700afccd484879ae6ddfd91b2c0402, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/292d6643cd4f402a97fe6610d23efdba] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.2 K 2024-11-10T17:23:38,414 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting a97b6ad5018e4b70a22f2639ff55c167, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731259415297 2024-11-10T17:23:38,415 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c700afccd484879ae6ddfd91b2c0402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731259415328 2024-11-10T17:23:38,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742186_1362 (size=4469) 2024-11-10T17:23:38,416 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 292d6643cd4f402a97fe6610d23efdba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259417547 2024-11-10T17:23:38,417 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#A#compaction#307 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:38,418 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/0c5920f753034bc3b269fdb257002c87 is 175, key is test_row_0/A:col10/1731259417547/Put/seqid=0 2024-11-10T17:23:38,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110688693ea4f1847eb87d763abf6174d1e_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259418259/Put/seqid=0 2024-11-10T17:23:38,452 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:38,453 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/4feffe30e35d4db195b655a5f3ecee4c is 50, key is test_row_0/C:col10/1731259417547/Put/seqid=0 2024-11-10T17:23:38,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259478449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259478449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259478457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742188_1364 (size=14594) 2024-11-10T17:23:38,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742187_1363 (size=31058) 2024-11-10T17:23:38,475 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/0c5920f753034bc3b269fdb257002c87 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/0c5920f753034bc3b269fdb257002c87 2024-11-10T17:23:38,482 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/A of a43b207a293e67a9738e21c91e1dc09e into 0c5920f753034bc3b269fdb257002c87(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:38,483 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:38,483 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/A, priority=13, startTime=1731259418349; duration=0sec 2024-11-10T17:23:38,483 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:38,483 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:A 2024-11-10T17:23:38,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742189_1365 (size=12104) 2024-11-10T17:23:38,501 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/4feffe30e35d4db195b655a5f3ecee4c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4feffe30e35d4db195b655a5f3ecee4c 2024-11-10T17:23:38,506 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/C of a43b207a293e67a9738e21c91e1dc09e into 4feffe30e35d4db195b655a5f3ecee4c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:38,506 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:38,507 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/C, priority=13, startTime=1731259418350; duration=0sec 2024-11-10T17:23:38,507 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:38,507 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:C 2024-11-10T17:23:38,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259478572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259478572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259478572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259478782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259478782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259478782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:38,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:38,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:38,868 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:38,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110688693ea4f1847eb87d763abf6174d1e_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110688693ea4f1847eb87d763abf6174d1e_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:38,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7ecd10dc57cd465a84ae76d6bb5ad56f, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:38,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7ecd10dc57cd465a84ae76d6bb5ad56f is 175, key is test_row_0/A:col10/1731259418259/Put/seqid=0 2024-11-10T17:23:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742190_1366 (size=39549) 2024-11-10T17:23:38,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7ecd10dc57cd465a84ae76d6bb5ad56f 2024-11-10T17:23:38,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/29ca2b5408e3482781eb4ab35c7f4ab2 is 50, key is test_row_0/B:col10/1731259418259/Put/seqid=0 2024-11-10T17:23:38,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742191_1367 (size=12001) 2024-11-10T17:23:38,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/29ca2b5408e3482781eb4ab35c7f4ab2 2024-11-10T17:23:38,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/61ab69714ab64bfdbf0c892522a68bae is 50, key is test_row_0/C:col10/1731259418259/Put/seqid=0 2024-11-10T17:23:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742192_1368 (size=12001) 2024-11-10T17:23:38,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/61ab69714ab64bfdbf0c892522a68bae 2024-11-10T17:23:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7ecd10dc57cd465a84ae76d6bb5ad56f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7ecd10dc57cd465a84ae76d6bb5ad56f 2024-11-10T17:23:38,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7ecd10dc57cd465a84ae76d6bb5ad56f, entries=200, sequenceid=77, filesize=38.6 K 2024-11-10T17:23:38,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/29ca2b5408e3482781eb4ab35c7f4ab2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/29ca2b5408e3482781eb4ab35c7f4ab2 2024-11-10T17:23:38,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/29ca2b5408e3482781eb4ab35c7f4ab2, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T17:23:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/61ab69714ab64bfdbf0c892522a68bae as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/61ab69714ab64bfdbf0c892522a68bae 2024-11-10T17:23:38,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/61ab69714ab64bfdbf0c892522a68bae, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T17:23:38,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a43b207a293e67a9738e21c91e1dc09e in 556ms, sequenceid=77, compaction requested=false 2024-11-10T17:23:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:39,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T17:23:39,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110769e3e49d2b247ec9835df2e1f8018e0_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259418430/Put/seqid=0 2024-11-10T17:23:39,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742193_1369 (size=14594) 2024-11-10T17:23:39,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259479153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259479155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259479160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259479272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259479272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259479272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-10T17:23:39,442 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:39,442 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-10T17:23:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T17:23:39,444 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:39,445 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:39,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259479491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259479491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259479491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,516 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:39,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110769e3e49d2b247ec9835df2e1f8018e0_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110769e3e49d2b247ec9835df2e1f8018e0_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:39,521 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/82b9b8af007a4e12841ddb0ce4ec3f89, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:39,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/82b9b8af007a4e12841ddb0ce4ec3f89 is 175, key is test_row_0/A:col10/1731259418430/Put/seqid=0 2024-11-10T17:23:39,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742194_1370 (size=39549) 2024-11-10T17:23:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T17:23:39,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259479561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,571 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:39,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259479581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,583 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:39,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:39,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T17:23:39,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259479811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259479811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259479811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:39,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:39,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:39,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:39,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/82b9b8af007a4e12841ddb0ce4ec3f89 2024-11-10T17:23:39,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e13381208e444137bb06c7d837fcd9ba is 50, key is test_row_0/B:col10/1731259418430/Put/seqid=0 2024-11-10T17:23:39,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742195_1371 (size=12001) 2024-11-10T17:23:40,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T17:23:40,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:40,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259480321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:40,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e13381208e444137bb06c7d837fcd9ba 2024-11-10T17:23:40,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259480332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:40,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259480332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:40,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/8f9039535a9248d882f24f90918bffbc is 50, key is test_row_0/C:col10/1731259418430/Put/seqid=0 2024-11-10T17:23:40,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742196_1372 (size=12001) 2024-11-10T17:23:40,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:40,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:40,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:40,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T17:23:40,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:40,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/8f9039535a9248d882f24f90918bffbc 2024-11-10T17:23:40,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/82b9b8af007a4e12841ddb0ce4ec3f89 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/82b9b8af007a4e12841ddb0ce4ec3f89 2024-11-10T17:23:40,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/82b9b8af007a4e12841ddb0ce4ec3f89, entries=200, sequenceid=95, filesize=38.6 K 2024-11-10T17:23:40,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e13381208e444137bb06c7d837fcd9ba as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e13381208e444137bb06c7d837fcd9ba 2024-11-10T17:23:40,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e13381208e444137bb06c7d837fcd9ba, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T17:23:40,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/8f9039535a9248d882f24f90918bffbc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8f9039535a9248d882f24f90918bffbc 2024-11-10T17:23:40,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8f9039535a9248d882f24f90918bffbc, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T17:23:40,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a43b207a293e67a9738e21c91e1dc09e in 1683ms, sequenceid=95, compaction requested=true 2024-11-10T17:23:40,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:40,783 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:40,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:40,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:40,786 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:40,786 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:40,786 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/A is initiating minor compaction (all files) 2024-11-10T17:23:40,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:40,786 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/A in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,786 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/0c5920f753034bc3b269fdb257002c87, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7ecd10dc57cd465a84ae76d6bb5ad56f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/82b9b8af007a4e12841ddb0ce4ec3f89] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=107.6 K 2024-11-10T17:23:40,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:40,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:40,786 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:40,786 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/0c5920f753034bc3b269fdb257002c87, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7ecd10dc57cd465a84ae76d6bb5ad56f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/82b9b8af007a4e12841ddb0ce4ec3f89] 2024-11-10T17:23:40,787 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c5920f753034bc3b269fdb257002c87, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259417547 2024-11-10T17:23:40,787 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ecd10dc57cd465a84ae76d6bb5ad56f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731259418259 2024-11-10T17:23:40,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:40,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/B is initiating minor compaction (all files) 2024-11-10T17:23:40,787 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82b9b8af007a4e12841ddb0ce4ec3f89, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731259418430 2024-11-10T17:23:40,787 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/B in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,788 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/4066066c71b0498a820321f150c29058, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/29ca2b5408e3482781eb4ab35c7f4ab2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e13381208e444137bb06c7d837fcd9ba] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.3 K 2024-11-10T17:23:40,788 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4066066c71b0498a820321f150c29058, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259417547 2024-11-10T17:23:40,790 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ca2b5408e3482781eb4ab35c7f4ab2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731259418259 2024-11-10T17:23:40,790 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e13381208e444137bb06c7d837fcd9ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731259418430 2024-11-10T17:23:40,799 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:40,804 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:40,805 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/b1f37e832076453881ec82af3652985f is 50, key is test_row_0/B:col10/1731259418430/Put/seqid=0 2024-11-10T17:23:40,805 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ac5a9bc3ebe14e5b8f3305f729d13471_a43b207a293e67a9738e21c91e1dc09e store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:40,807 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ac5a9bc3ebe14e5b8f3305f729d13471_a43b207a293e67a9738e21c91e1dc09e, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:40,807 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ac5a9bc3ebe14e5b8f3305f729d13471_a43b207a293e67a9738e21c91e1dc09e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:40,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742197_1373 (size=12207) 2024-11-10T17:23:40,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T17:23:40,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:40,825 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T17:23:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742198_1374 (size=4469) 2024-11-10T17:23:40,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104dfcb337490b4029a38caade4a1c927e_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259419151/Put/seqid=0 2024-11-10T17:23:40,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742199_1375 (size=12154) 2024-11-10T17:23:41,226 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/b1f37e832076453881ec82af3652985f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b1f37e832076453881ec82af3652985f 2024-11-10T17:23:41,236 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/B of a43b207a293e67a9738e21c91e1dc09e into b1f37e832076453881ec82af3652985f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:41,236 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:41,236 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/B, priority=13, startTime=1731259420786; duration=0sec 2024-11-10T17:23:41,236 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:41,236 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:B 2024-11-10T17:23:41,236 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:41,237 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:41,237 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/C is initiating minor compaction (all files) 2024-11-10T17:23:41,238 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/C in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:41,238 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:41,238 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/59fb64847aa54bd198a6bd6bdf2ab187 is 175, key is test_row_0/A:col10/1731259418430/Put/seqid=0 2024-11-10T17:23:41,239 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4feffe30e35d4db195b655a5f3ecee4c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/61ab69714ab64bfdbf0c892522a68bae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8f9039535a9248d882f24f90918bffbc] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.3 K 2024-11-10T17:23:41,240 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4feffe30e35d4db195b655a5f3ecee4c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259417547 2024-11-10T17:23:41,242 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 61ab69714ab64bfdbf0c892522a68bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731259418259 2024-11-10T17:23:41,242 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f9039535a9248d882f24f90918bffbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731259418430 2024-11-10T17:23:41,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742200_1376 (size=31161) 2024-11-10T17:23:41,252 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:41,252 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/8fbebd65a422437585cde8ca79b4107b is 50, key is test_row_0/C:col10/1731259418430/Put/seqid=0 2024-11-10T17:23:41,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:41,257 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104dfcb337490b4029a38caade4a1c927e_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104dfcb337490b4029a38caade4a1c927e_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:41,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742201_1377 (size=12207) 2024-11-10T17:23:41,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5c456c70b7c749988ead666df0b91b93, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:41,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5c456c70b7c749988ead666df0b91b93 is 175, key is test_row_0/A:col10/1731259419151/Put/seqid=0 2024-11-10T17:23:41,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742202_1378 (size=30955) 2024-11-10T17:23:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:41,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259481385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259481386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259481387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259481502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259481502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259481502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T17:23:41,649 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/59fb64847aa54bd198a6bd6bdf2ab187 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/59fb64847aa54bd198a6bd6bdf2ab187 2024-11-10T17:23:41,653 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/A of a43b207a293e67a9738e21c91e1dc09e into 59fb64847aa54bd198a6bd6bdf2ab187(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:41,653 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:41,653 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/A, priority=13, startTime=1731259420783; duration=0sec 2024-11-10T17:23:41,653 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:41,654 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:A 2024-11-10T17:23:41,663 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5c456c70b7c749988ead666df0b91b93 2024-11-10T17:23:41,663 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/8fbebd65a422437585cde8ca79b4107b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8fbebd65a422437585cde8ca79b4107b 2024-11-10T17:23:41,668 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/C of a43b207a293e67a9738e21c91e1dc09e into 8fbebd65a422437585cde8ca79b4107b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:41,668 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:41,668 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/C, priority=13, startTime=1731259420786; duration=0sec 2024-11-10T17:23:41,668 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:41,668 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:C 2024-11-10T17:23:41,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/6878a83074534900a4c20a42f9d35c02 is 50, key is test_row_0/B:col10/1731259419151/Put/seqid=0 2024-11-10T17:23:41,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742203_1379 (size=12001) 2024-11-10T17:23:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259481712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259481721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259481721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259482031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259482042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259482042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,076 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/6878a83074534900a4c20a42f9d35c02 2024-11-10T17:23:42,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3278eb4edc9840b6856b7ab175e336b5 is 50, key is test_row_0/C:col10/1731259419151/Put/seqid=0 2024-11-10T17:23:42,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742204_1380 (size=12001) 2024-11-10T17:23:42,488 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3278eb4edc9840b6856b7ab175e336b5 2024-11-10T17:23:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/5c456c70b7c749988ead666df0b91b93 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5c456c70b7c749988ead666df0b91b93 2024-11-10T17:23:42,497 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5c456c70b7c749988ead666df0b91b93, entries=150, sequenceid=116, filesize=30.2 K 2024-11-10T17:23:42,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/6878a83074534900a4c20a42f9d35c02 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/6878a83074534900a4c20a42f9d35c02 2024-11-10T17:23:42,502 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/6878a83074534900a4c20a42f9d35c02, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T17:23:42,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3278eb4edc9840b6856b7ab175e336b5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3278eb4edc9840b6856b7ab175e336b5 2024-11-10T17:23:42,508 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3278eb4edc9840b6856b7ab175e336b5, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T17:23:42,509 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a43b207a293e67a9738e21c91e1dc09e in 1684ms, sequenceid=116, compaction requested=false 2024-11-10T17:23:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-10T17:23:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-10T17:23:42,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-10T17:23:42,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0660 sec 2024-11-10T17:23:42,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.0710 sec 2024-11-10T17:23:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:42,548 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:42,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T17:23:42,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107b87c69d9f4e4b0ab446fb8c56de61b6_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259422547/Put/seqid=0 2024-11-10T17:23:42,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742205_1381 (size=14794) 2024-11-10T17:23:42,562 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:42,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107b87c69d9f4e4b0ab446fb8c56de61b6_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b87c69d9f4e4b0ab446fb8c56de61b6_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:42,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/8912d8cfc66c497fa2292248a75956c0, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:42,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/8912d8cfc66c497fa2292248a75956c0 is 175, key is test_row_0/A:col10/1731259422547/Put/seqid=0 2024-11-10T17:23:42,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742206_1382 (size=39749) 2024-11-10T17:23:42,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259482603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259482608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259482610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259482711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259482722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259482732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259482931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259482942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259482952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:42,973 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/8912d8cfc66c497fa2292248a75956c0 2024-11-10T17:23:42,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/2b31bcdb5d0644cca7e5bc472b32daca is 50, key is test_row_0/B:col10/1731259422547/Put/seqid=0 2024-11-10T17:23:42,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742207_1383 (size=12151) 2024-11-10T17:23:43,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259483251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259483261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259483272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/2b31bcdb5d0644cca7e5bc472b32daca 2024-11-10T17:23:43,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e50ec84c56b9458499270de87888d832 is 50, key is test_row_0/C:col10/1731259422547/Put/seqid=0 2024-11-10T17:23:43,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742208_1384 (size=12151) 2024-11-10T17:23:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T17:23:43,602 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:43,602 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-10T17:23:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T17:23:43,604 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:43,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:43,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259483601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,611 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259483622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,629 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T17:23:43,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-10T17:23:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:43,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:43,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259483772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259483781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259483781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:43,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e50ec84c56b9458499270de87888d832 2024-11-10T17:23:43,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/8912d8cfc66c497fa2292248a75956c0 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/8912d8cfc66c497fa2292248a75956c0 2024-11-10T17:23:43,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/8912d8cfc66c497fa2292248a75956c0, entries=200, sequenceid=135, filesize=38.8 K 2024-11-10T17:23:43,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/2b31bcdb5d0644cca7e5bc472b32daca as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/2b31bcdb5d0644cca7e5bc472b32daca 2024-11-10T17:23:43,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/2b31bcdb5d0644cca7e5bc472b32daca, entries=150, sequenceid=135, filesize=11.9 K 2024-11-10T17:23:43,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e50ec84c56b9458499270de87888d832 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e50ec84c56b9458499270de87888d832 2024-11-10T17:23:43,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e50ec84c56b9458499270de87888d832, entries=150, sequenceid=135, filesize=11.9 K 2024-11-10T17:23:43,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a43b207a293e67a9738e21c91e1dc09e in 1276ms, sequenceid=135, compaction requested=true 2024-11-10T17:23:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:43,825 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:43,825 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:43,827 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:43,827 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/A is initiating minor compaction (all files) 2024-11-10T17:23:43,827 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/A in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:43,827 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/59fb64847aa54bd198a6bd6bdf2ab187, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5c456c70b7c749988ead666df0b91b93, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/8912d8cfc66c497fa2292248a75956c0] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=99.5 K 2024-11-10T17:23:43,827 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:43,827 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/59fb64847aa54bd198a6bd6bdf2ab187, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5c456c70b7c749988ead666df0b91b93, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/8912d8cfc66c497fa2292248a75956c0] 2024-11-10T17:23:43,828 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59fb64847aa54bd198a6bd6bdf2ab187, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731259418430 2024-11-10T17:23:43,828 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c456c70b7c749988ead666df0b91b93, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731259419150 2024-11-10T17:23:43,829 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8912d8cfc66c497fa2292248a75956c0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731259421383 2024-11-10T17:23:43,829 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:43,829 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/B is initiating minor compaction (all files) 2024-11-10T17:23:43,829 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/B in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:43,829 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b1f37e832076453881ec82af3652985f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/6878a83074534900a4c20a42f9d35c02, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/2b31bcdb5d0644cca7e5bc472b32daca] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.5 K 2024-11-10T17:23:43,829 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f37e832076453881ec82af3652985f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731259418430 2024-11-10T17:23:43,830 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6878a83074534900a4c20a42f9d35c02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731259419150 2024-11-10T17:23:43,830 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b31bcdb5d0644cca7e5bc472b32daca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731259421383 2024-11-10T17:23:43,838 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:43,838 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/5fee10f8b89b4c1ab8f2389e1cf11b96 is 50, key is test_row_0/B:col10/1731259422547/Put/seqid=0 2024-11-10T17:23:43,841 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:43,850 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110843bd2d9b4b84d3294fde0a75d0ca89a_a43b207a293e67a9738e21c91e1dc09e store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:43,851 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110843bd2d9b4b84d3294fde0a75d0ca89a_a43b207a293e67a9738e21c91e1dc09e, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:43,851 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110843bd2d9b4b84d3294fde0a75d0ca89a_a43b207a293e67a9738e21c91e1dc09e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:43,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742209_1385 (size=12459) 2024-11-10T17:23:43,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742210_1386 (size=4469) 2024-11-10T17:23:43,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-10T17:23:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:43,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T17:23:43,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110149a9e263b974df4961ead3fcbf9bd75_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259422608/Put/seqid=0 2024-11-10T17:23:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T17:23:43,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742211_1387 (size=12304) 2024-11-10T17:23:43,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:43,948 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110149a9e263b974df4961ead3fcbf9bd75_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110149a9e263b974df4961ead3fcbf9bd75_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:43,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/1bdd58d6764e40b899746fccd7211017, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:43,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/1bdd58d6764e40b899746fccd7211017 is 175, key is test_row_0/A:col10/1731259422608/Put/seqid=0 2024-11-10T17:23:43,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742212_1388 (size=31105) 2024-11-10T17:23:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T17:23:44,261 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/5fee10f8b89b4c1ab8f2389e1cf11b96 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/5fee10f8b89b4c1ab8f2389e1cf11b96 2024-11-10T17:23:44,265 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:44,266 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/4b8e680a69dc46d7b519754e4d04e4c4 is 175, key is test_row_0/A:col10/1731259422547/Put/seqid=0 2024-11-10T17:23:44,266 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/B of a43b207a293e67a9738e21c91e1dc09e into 5fee10f8b89b4c1ab8f2389e1cf11b96(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:44,266 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:44,266 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/B, priority=13, startTime=1731259423825; duration=0sec 2024-11-10T17:23:44,267 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:44,267 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:B 2024-11-10T17:23:44,267 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:44,268 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:44,268 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/C is initiating minor compaction (all files) 2024-11-10T17:23:44,268 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/C in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:44,268 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8fbebd65a422437585cde8ca79b4107b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3278eb4edc9840b6856b7ab175e336b5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e50ec84c56b9458499270de87888d832] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.5 K 2024-11-10T17:23:44,268 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fbebd65a422437585cde8ca79b4107b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731259418430 2024-11-10T17:23:44,269 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3278eb4edc9840b6856b7ab175e336b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731259419150 2024-11-10T17:23:44,269 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e50ec84c56b9458499270de87888d832, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731259421383 2024-11-10T17:23:44,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742213_1389 (size=31413) 2024-11-10T17:23:44,287 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:44,288 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a757f4d3ba2744cf94beefd28adb051d is 50, key is test_row_0/C:col10/1731259422547/Put/seqid=0 2024-11-10T17:23:44,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742214_1390 (size=12459) 2024-11-10T17:23:44,299 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a757f4d3ba2744cf94beefd28adb051d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a757f4d3ba2744cf94beefd28adb051d 2024-11-10T17:23:44,304 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/C of a43b207a293e67a9738e21c91e1dc09e into a757f4d3ba2744cf94beefd28adb051d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:44,304 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:44,304 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/C, priority=13, startTime=1731259423825; duration=0sec 2024-11-10T17:23:44,304 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:44,304 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:C 2024-11-10T17:23:44,359 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/1bdd58d6764e40b899746fccd7211017 2024-11-10T17:23:44,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/d0422cec4bc54bf49721b57f238b6b31 is 50, key is test_row_0/B:col10/1731259422608/Put/seqid=0 2024-11-10T17:23:44,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742215_1391 (size=12151) 2024-11-10T17:23:44,675 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/4b8e680a69dc46d7b519754e4d04e4c4 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/4b8e680a69dc46d7b519754e4d04e4c4 2024-11-10T17:23:44,679 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/A of a43b207a293e67a9738e21c91e1dc09e into 4b8e680a69dc46d7b519754e4d04e4c4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:44,679 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:44,679 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/A, priority=13, startTime=1731259423825; duration=0sec 2024-11-10T17:23:44,679 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:44,679 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:A 2024-11-10T17:23:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T17:23:44,773 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/d0422cec4bc54bf49721b57f238b6b31 2024-11-10T17:23:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:44,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e5b4150aaf7e4b1c9c665a4648fa8409 is 50, key is test_row_0/C:col10/1731259422608/Put/seqid=0 2024-11-10T17:23:44,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742216_1392 (size=12151) 2024-11-10T17:23:44,821 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e5b4150aaf7e4b1c9c665a4648fa8409 2024-11-10T17:23:44,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/1bdd58d6764e40b899746fccd7211017 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/1bdd58d6764e40b899746fccd7211017 2024-11-10T17:23:44,830 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/1bdd58d6764e40b899746fccd7211017, entries=150, sequenceid=155, filesize=30.4 K 2024-11-10T17:23:44,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/d0422cec4bc54bf49721b57f238b6b31 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d0422cec4bc54bf49721b57f238b6b31 2024-11-10T17:23:44,836 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d0422cec4bc54bf49721b57f238b6b31, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T17:23:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e5b4150aaf7e4b1c9c665a4648fa8409 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e5b4150aaf7e4b1c9c665a4648fa8409 2024-11-10T17:23:44,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259484834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:44,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:44,846 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e5b4150aaf7e4b1c9c665a4648fa8409, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T17:23:44,847 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a43b207a293e67a9738e21c91e1dc09e in 934ms, sequenceid=155, compaction requested=false 2024-11-10T17:23:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-10T17:23:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-10T17:23:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:44,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:44,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T17:23:44,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-10T17:23:44,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2420 sec 2024-11-10T17:23:44,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.2500 sec 2024-11-10T17:23:44,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c6b05db470842268fdeb6994716a270_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259424835/Put/seqid=0 2024-11-10T17:23:44,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742217_1393 (size=14794) 2024-11-10T17:23:44,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259484929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259484940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259484951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259485051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259485061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259485162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259485272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259485282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,297 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:45,302 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c6b05db470842268fdeb6994716a270_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c6b05db470842268fdeb6994716a270_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:45,302 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/6c14188104dd4152a9fb40485726f877, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:45,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/6c14188104dd4152a9fb40485726f877 is 175, key is test_row_0/A:col10/1731259424835/Put/seqid=0 2024-11-10T17:23:45,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742218_1394 (size=39749) 2024-11-10T17:23:45,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259485482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259485591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259485602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:45,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/6c14188104dd4152a9fb40485726f877 2024-11-10T17:23:45,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/751aac39039d4ee886070b84d7a75641 is 50, key is test_row_0/B:col10/1731259424835/Put/seqid=0 2024-11-10T17:23:45,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742219_1395 (size=12151) 2024-11-10T17:23:45,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/751aac39039d4ee886070b84d7a75641 2024-11-10T17:23:45,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a0f56aa9243f409696379826b85f767e is 50, key is test_row_0/C:col10/1731259424835/Put/seqid=0 2024-11-10T17:23:45,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742220_1396 (size=12151) 2024-11-10T17:23:45,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a0f56aa9243f409696379826b85f767e 2024-11-10T17:23:45,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/6c14188104dd4152a9fb40485726f877 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/6c14188104dd4152a9fb40485726f877 2024-11-10T17:23:45,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/6c14188104dd4152a9fb40485726f877, entries=200, sequenceid=175, filesize=38.8 K 2024-11-10T17:23:45,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/751aac39039d4ee886070b84d7a75641 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/751aac39039d4ee886070b84d7a75641 2024-11-10T17:23:45,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/751aac39039d4ee886070b84d7a75641, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T17:23:45,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/a0f56aa9243f409696379826b85f767e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a0f56aa9243f409696379826b85f767e 2024-11-10T17:23:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T17:23:45,752 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:45,752 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-10T17:23:45,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a0f56aa9243f409696379826b85f767e, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T17:23:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T17:23:45,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a43b207a293e67a9738e21c91e1dc09e in 906ms, sequenceid=175, compaction requested=true 2024-11-10T17:23:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:45,755 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:45,755 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:45,755 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:45,755 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:45,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:45,757 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:45,757 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/A is initiating minor compaction (all files) 2024-11-10T17:23:45,757 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/A in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:45,757 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/4b8e680a69dc46d7b519754e4d04e4c4, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/1bdd58d6764e40b899746fccd7211017, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/6c14188104dd4152a9fb40485726f877] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=99.9 K 2024-11-10T17:23:45,757 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:45,757 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/4b8e680a69dc46d7b519754e4d04e4c4, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/1bdd58d6764e40b899746fccd7211017, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/6c14188104dd4152a9fb40485726f877] 2024-11-10T17:23:45,758 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:45,758 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/B is initiating minor compaction (all files) 2024-11-10T17:23:45,758 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/B in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:45,758 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/5fee10f8b89b4c1ab8f2389e1cf11b96, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d0422cec4bc54bf49721b57f238b6b31, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/751aac39039d4ee886070b84d7a75641] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.9 K 2024-11-10T17:23:45,758 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8e680a69dc46d7b519754e4d04e4c4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731259421383 2024-11-10T17:23:45,759 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fee10f8b89b4c1ab8f2389e1cf11b96, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731259421383 2024-11-10T17:23:45,759 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bdd58d6764e40b899746fccd7211017, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731259422594 2024-11-10T17:23:45,759 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c14188104dd4152a9fb40485726f877, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731259424811 2024-11-10T17:23:45,759 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d0422cec4bc54bf49721b57f238b6b31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731259422594 2024-11-10T17:23:45,759 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 751aac39039d4ee886070b84d7a75641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731259424821 2024-11-10T17:23:45,765 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:45,768 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411107a47558ed3e8476989cbef7590cf2f80_a43b207a293e67a9738e21c91e1dc09e store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:45,769 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:45,770 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e15179f0da9a43bdaf5102ff0b9b3ae8 is 50, key is test_row_0/B:col10/1731259424835/Put/seqid=0 2024-11-10T17:23:45,770 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411107a47558ed3e8476989cbef7590cf2f80_a43b207a293e67a9738e21c91e1dc09e, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:45,770 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107a47558ed3e8476989cbef7590cf2f80_a43b207a293e67a9738e21c91e1dc09e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:45,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742222_1398 (size=4469) 2024-11-10T17:23:45,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742221_1397 (size=12561) 2024-11-10T17:23:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T17:23:45,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T17:23:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:45,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T17:23:45,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110acfdfc576fb14cba94b0a13ae8ddd1a9_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259424926/Put/seqid=0 2024-11-10T17:23:45,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742223_1399 (size=12304) 2024-11-10T17:23:45,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:45,936 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110acfdfc576fb14cba94b0a13ae8ddd1a9_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110acfdfc576fb14cba94b0a13ae8ddd1a9_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:45,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7a538739675843e9988e91c2cd4f2222, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7a538739675843e9988e91c2cd4f2222 is 175, key is test_row_0/A:col10/1731259424926/Put/seqid=0 2024-11-10T17:23:45,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742224_1400 (size=31105) 2024-11-10T17:23:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:46,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T17:23:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259486121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259486123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259486124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,183 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:46,184 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/75c3e617597f43dca39c9397e54a5a18 is 175, key is test_row_0/A:col10/1731259424835/Put/seqid=0 2024-11-10T17:23:46,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742225_1401 (size=31515) 2024-11-10T17:23:46,192 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/75c3e617597f43dca39c9397e54a5a18 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/75c3e617597f43dca39c9397e54a5a18 2024-11-10T17:23:46,195 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/A of a43b207a293e67a9738e21c91e1dc09e into 75c3e617597f43dca39c9397e54a5a18(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:46,195 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:46,196 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/A, priority=13, startTime=1731259425755; duration=0sec 2024-11-10T17:23:46,196 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:46,196 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:A 2024-11-10T17:23:46,196 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:23:46,197 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:23:46,197 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/C is initiating minor compaction (all files) 2024-11-10T17:23:46,197 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/C in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:46,197 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a757f4d3ba2744cf94beefd28adb051d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e5b4150aaf7e4b1c9c665a4648fa8409, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a0f56aa9243f409696379826b85f767e] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=35.9 K 2024-11-10T17:23:46,197 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting a757f4d3ba2744cf94beefd28adb051d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731259421383 2024-11-10T17:23:46,198 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b4150aaf7e4b1c9c665a4648fa8409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731259422594 2024-11-10T17:23:46,198 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f56aa9243f409696379826b85f767e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731259424821 2024-11-10T17:23:46,202 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e15179f0da9a43bdaf5102ff0b9b3ae8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e15179f0da9a43bdaf5102ff0b9b3ae8 2024-11-10T17:23:46,207 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:46,208 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/B of a43b207a293e67a9738e21c91e1dc09e into e15179f0da9a43bdaf5102ff0b9b3ae8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:46,208 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:46,208 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/dc2309d64cd34b58a81877bf37b42931 is 50, key is test_row_0/C:col10/1731259424835/Put/seqid=0 2024-11-10T17:23:46,208 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/B, priority=13, startTime=1731259425755; duration=0sec 2024-11-10T17:23:46,208 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:46,208 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:B 2024-11-10T17:23:46,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742226_1402 (size=12561) 2024-11-10T17:23:46,219 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/dc2309d64cd34b58a81877bf37b42931 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/dc2309d64cd34b58a81877bf37b42931 2024-11-10T17:23:46,223 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/C of a43b207a293e67a9738e21c91e1dc09e into dc2309d64cd34b58a81877bf37b42931(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:46,223 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:46,223 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/C, priority=13, startTime=1731259425755; duration=0sec 2024-11-10T17:23:46,223 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:46,223 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:C 2024-11-10T17:23:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259486241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259486241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,343 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7a538739675843e9988e91c2cd4f2222 2024-11-10T17:23:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/0928162543a949ef91bb0855c985e092 is 50, key is test_row_0/B:col10/1731259424926/Put/seqid=0 2024-11-10T17:23:46,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742227_1403 (size=12151) 2024-11-10T17:23:46,357 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/0928162543a949ef91bb0855c985e092 2024-11-10T17:23:46,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e26b589170f5446c8db7eba5bf840387 is 50, key is test_row_0/C:col10/1731259424926/Put/seqid=0 2024-11-10T17:23:46,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742228_1404 (size=12151) 2024-11-10T17:23:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T17:23:46,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259486461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259486461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,773 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e26b589170f5446c8db7eba5bf840387 2024-11-10T17:23:46,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259486772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259486772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:46,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7a538739675843e9988e91c2cd4f2222 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7a538739675843e9988e91c2cd4f2222 2024-11-10T17:23:46,781 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7a538739675843e9988e91c2cd4f2222, entries=150, sequenceid=194, filesize=30.4 K 2024-11-10T17:23:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/0928162543a949ef91bb0855c985e092 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0928162543a949ef91bb0855c985e092 2024-11-10T17:23:46,785 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0928162543a949ef91bb0855c985e092, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T17:23:46,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/e26b589170f5446c8db7eba5bf840387 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e26b589170f5446c8db7eba5bf840387 2024-11-10T17:23:46,789 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e26b589170f5446c8db7eba5bf840387, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T17:23:46,790 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a43b207a293e67a9738e21c91e1dc09e in 882ms, sequenceid=194, compaction requested=false 2024-11-10T17:23:46,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:46,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:46,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-10T17:23:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-10T17:23:46,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-10T17:23:46,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-10T17:23:46,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.0420 sec 2024-11-10T17:23:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T17:23:46,892 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:46,893 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-10T17:23:46,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T17:23:46,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:46,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T17:23:47,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T17:23:47,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:47,053 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T17:23:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110536ca255e4ef4930a56e498988e2e79d_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259426098/Put/seqid=0 2024-11-10T17:23:47,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742229_1405 (size=12304) 2024-11-10T17:23:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:47,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T17:23:47,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259487277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:47,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259487282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:47,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259487282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259487401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:47,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:47,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110536ca255e4ef4930a56e498988e2e79d_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110536ca255e4ef4930a56e498988e2e79d_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:47,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e940f5ba72974f0a9e9a868c6844d4ed, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:47,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e940f5ba72974f0a9e9a868c6844d4ed is 175, key is test_row_0/A:col10/1731259426098/Put/seqid=0 2024-11-10T17:23:47,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742230_1406 (size=31105) 2024-11-10T17:23:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T17:23:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259487611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:47,882 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e940f5ba72974f0a9e9a868c6844d4ed 2024-11-10T17:23:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/de83099b51c647a8ad49c1d55d423380 is 50, key is test_row_0/B:col10/1731259426098/Put/seqid=0 2024-11-10T17:23:47,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742231_1407 (size=12151) 2024-11-10T17:23:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259487922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T17:23:48,294 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/de83099b51c647a8ad49c1d55d423380 2024-11-10T17:23:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/70625b480cfa40779512dfc2e51dda7e is 50, key is test_row_0/C:col10/1731259426098/Put/seqid=0 2024-11-10T17:23:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742232_1408 (size=12151) 2024-11-10T17:23:48,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259488301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:48,311 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:48,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259488312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:48,321 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:48,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259488432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:48,706 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/70625b480cfa40779512dfc2e51dda7e 2024-11-10T17:23:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e940f5ba72974f0a9e9a868c6844d4ed as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e940f5ba72974f0a9e9a868c6844d4ed 2024-11-10T17:23:48,716 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e940f5ba72974f0a9e9a868c6844d4ed, entries=150, sequenceid=214, filesize=30.4 K 2024-11-10T17:23:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/de83099b51c647a8ad49c1d55d423380 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/de83099b51c647a8ad49c1d55d423380 2024-11-10T17:23:48,721 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/de83099b51c647a8ad49c1d55d423380, entries=150, sequenceid=214, filesize=11.9 K 2024-11-10T17:23:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/70625b480cfa40779512dfc2e51dda7e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/70625b480cfa40779512dfc2e51dda7e 2024-11-10T17:23:48,726 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/70625b480cfa40779512dfc2e51dda7e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-10T17:23:48,727 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a43b207a293e67a9738e21c91e1dc09e in 1674ms, sequenceid=214, compaction requested=true 2024-11-10T17:23:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-10T17:23:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-10T17:23:48,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-10T17:23:48,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-10T17:23:48,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.8390 sec 2024-11-10T17:23:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T17:23:49,042 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:49,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-10T17:23:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T17:23:49,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:49,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:49,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T17:23:49,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T17:23:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:49,198 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T17:23:49,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111085db11d11e934d74891635c8851a9b64_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259427255/Put/seqid=0 2024-11-10T17:23:49,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742233_1409 (size=12304) 2024-11-10T17:23:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T17:23:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:49,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:49,616 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111085db11d11e934d74891635c8851a9b64_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111085db11d11e934d74891635c8851a9b64_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:49,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259489608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:49,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/598d8272d03741eaad47572523aaf726, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/598d8272d03741eaad47572523aaf726 is 175, key is test_row_0/A:col10/1731259427255/Put/seqid=0 2024-11-10T17:23:49,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742234_1410 (size=31105) 2024-11-10T17:23:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T17:23:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259489722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259489941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:50,032 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/598d8272d03741eaad47572523aaf726 2024-11-10T17:23:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/da109ba9762c46bb916b5fe2e0a1877a is 50, key is test_row_0/B:col10/1731259427255/Put/seqid=0 2024-11-10T17:23:50,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742235_1411 (size=12151) 2024-11-10T17:23:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T17:23:50,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:50,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259490262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259490332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:50,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259490332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:50,344 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:50,344 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:50,448 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/da109ba9762c46bb916b5fe2e0a1877a 2024-11-10T17:23:50,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3f15d5ad00b44ac08bd247c9111f27a5 is 50, key is test_row_0/C:col10/1731259427255/Put/seqid=0 2024-11-10T17:23:50,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742236_1412 (size=12151) 2024-11-10T17:23:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259490782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:50,863 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3f15d5ad00b44ac08bd247c9111f27a5 2024-11-10T17:23:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/598d8272d03741eaad47572523aaf726 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/598d8272d03741eaad47572523aaf726 2024-11-10T17:23:50,875 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/598d8272d03741eaad47572523aaf726, entries=150, sequenceid=233, filesize=30.4 K 2024-11-10T17:23:50,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/da109ba9762c46bb916b5fe2e0a1877a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/da109ba9762c46bb916b5fe2e0a1877a 2024-11-10T17:23:50,881 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/da109ba9762c46bb916b5fe2e0a1877a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T17:23:50,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3f15d5ad00b44ac08bd247c9111f27a5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3f15d5ad00b44ac08bd247c9111f27a5 2024-11-10T17:23:50,885 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3f15d5ad00b44ac08bd247c9111f27a5, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T17:23:50,886 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a43b207a293e67a9738e21c91e1dc09e in 1688ms, sequenceid=233, compaction requested=true 2024-11-10T17:23:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-10T17:23:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-10T17:23:50,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-10T17:23:50,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-11-10T17:23:50,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8470 sec 2024-11-10T17:23:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T17:23:51,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:51,193 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-10T17:23:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T17:23:51,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:51,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:51,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T17:23:51,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T17:23:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:51,352 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T17:23:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b1cc64221f0e4291a653b28134cf1416_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259429593/Put/seqid=0 2024-11-10T17:23:51,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742237_1413 (size=12304) 2024-11-10T17:23:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T17:23:51,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:51,767 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b1cc64221f0e4291a653b28134cf1416_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1cc64221f0e4291a653b28134cf1416_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:51,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/46e2a310eb754077b9fe891d004d603f, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/46e2a310eb754077b9fe891d004d603f is 175, key is test_row_0/A:col10/1731259429593/Put/seqid=0 2024-11-10T17:23:51,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742238_1414 (size=31105) 2024-11-10T17:23:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T17:23:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259491964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:52,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259492072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:52,173 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/46e2a310eb754077b9fe891d004d603f 2024-11-10T17:23:52,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/d3c8c64236d540f28c3330aa9f702549 is 50, key is test_row_0/B:col10/1731259429593/Put/seqid=0 2024-11-10T17:23:52,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742239_1415 (size=12151) 2024-11-10T17:23:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259492291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T17:23:52,585 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/d3c8c64236d540f28c3330aa9f702549 2024-11-10T17:23:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/2c838054ae3e4eedabb913959d5fcda8 is 50, key is test_row_0/C:col10/1731259429593/Put/seqid=0 2024-11-10T17:23:52,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742240_1416 (size=12151) 2024-11-10T17:23:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259492612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:52,996 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/2c838054ae3e4eedabb913959d5fcda8 2024-11-10T17:23:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/46e2a310eb754077b9fe891d004d603f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/46e2a310eb754077b9fe891d004d603f 2024-11-10T17:23:53,006 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/46e2a310eb754077b9fe891d004d603f, entries=150, sequenceid=250, filesize=30.4 K 2024-11-10T17:23:53,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/d3c8c64236d540f28c3330aa9f702549 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d3c8c64236d540f28c3330aa9f702549 2024-11-10T17:23:53,010 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d3c8c64236d540f28c3330aa9f702549, entries=150, sequenceid=250, filesize=11.9 K 2024-11-10T17:23:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/2c838054ae3e4eedabb913959d5fcda8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/2c838054ae3e4eedabb913959d5fcda8 2024-11-10T17:23:53,014 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/2c838054ae3e4eedabb913959d5fcda8, entries=150, sequenceid=250, filesize=11.9 K 2024-11-10T17:23:53,015 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a43b207a293e67a9738e21c91e1dc09e in 1663ms, sequenceid=250, compaction requested=true 2024-11-10T17:23:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-10T17:23:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-10T17:23:53,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T17:23:53,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8220 sec 2024-11-10T17:23:53,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8260 sec 2024-11-10T17:23:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:53,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:53,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T17:23:53,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ab83ed0890f94897b78cc633eb1dcf69_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259433134/Put/seqid=0 2024-11-10T17:23:53,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742241_1417 (size=14994) 2024-11-10T17:23:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259493298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T17:23:53,342 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:53,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:23:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-10T17:23:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T17:23:53,344 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:23:53,345 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:23:53,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:23:53,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259493412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T17:23:53,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,547 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:53,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ab83ed0890f94897b78cc633eb1dcf69_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab83ed0890f94897b78cc633eb1dcf69_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:53,552 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e668525b516b4194b3210ccd76943d44, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:53,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e668525b516b4194b3210ccd76943d44 is 175, key is test_row_0/A:col10/1731259433134/Put/seqid=0 2024-11-10T17:23:53,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742242_1418 (size=39949) 2024-11-10T17:23:53,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e668525b516b4194b3210ccd76943d44 2024-11-10T17:23:53,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/ebbab6c50c234b0baa6e2ba27b511f37 is 50, key is test_row_0/B:col10/1731259433134/Put/seqid=0 2024-11-10T17:23:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742243_1419 (size=12301) 2024-11-10T17:23:53,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/ebbab6c50c234b0baa6e2ba27b511f37 2024-11-10T17:23:53,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/045a168222c044c785c58818c28e3898 is 50, key is test_row_0/C:col10/1731259433134/Put/seqid=0 2024-11-10T17:23:53,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742244_1420 (size=12301) 2024-11-10T17:23:53,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/045a168222c044c785c58818c28e3898 2024-11-10T17:23:53,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/e668525b516b4194b3210ccd76943d44 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e668525b516b4194b3210ccd76943d44 2024-11-10T17:23:53,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e668525b516b4194b3210ccd76943d44, entries=200, sequenceid=270, filesize=39.0 K 2024-11-10T17:23:53,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/ebbab6c50c234b0baa6e2ba27b511f37 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/ebbab6c50c234b0baa6e2ba27b511f37 2024-11-10T17:23:53,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/ebbab6c50c234b0baa6e2ba27b511f37, entries=150, sequenceid=270, filesize=12.0 K 2024-11-10T17:23:53,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/045a168222c044c785c58818c28e3898 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/045a168222c044c785c58818c28e3898 2024-11-10T17:23:53,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/045a168222c044c785c58818c28e3898, entries=150, sequenceid=270, filesize=12.0 K 2024-11-10T17:23:53,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a43b207a293e67a9738e21c91e1dc09e in 474ms, sequenceid=270, compaction requested=true 2024-11-10T17:23:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:53,610 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T17:23:53,610 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T17:23:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:53,612 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195884 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T17:23:53,612 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/A is initiating minor compaction (all files) 2024-11-10T17:23:53,612 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/A in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,613 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/75c3e617597f43dca39c9397e54a5a18, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7a538739675843e9988e91c2cd4f2222, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e940f5ba72974f0a9e9a868c6844d4ed, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/598d8272d03741eaad47572523aaf726, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/46e2a310eb754077b9fe891d004d603f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e668525b516b4194b3210ccd76943d44] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=191.3 K 2024-11-10T17:23:53,613 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,613 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/75c3e617597f43dca39c9397e54a5a18, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7a538739675843e9988e91c2cd4f2222, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e940f5ba72974f0a9e9a868c6844d4ed, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/598d8272d03741eaad47572523aaf726, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/46e2a310eb754077b9fe891d004d603f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e668525b516b4194b3210ccd76943d44] 2024-11-10T17:23:53,613 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73466 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T17:23:53,613 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c3e617597f43dca39c9397e54a5a18, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731259424821 2024-11-10T17:23:53,613 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/B is initiating minor compaction (all files) 2024-11-10T17:23:53,613 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/B in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,613 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e15179f0da9a43bdaf5102ff0b9b3ae8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0928162543a949ef91bb0855c985e092, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/de83099b51c647a8ad49c1d55d423380, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/da109ba9762c46bb916b5fe2e0a1877a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d3c8c64236d540f28c3330aa9f702549, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/ebbab6c50c234b0baa6e2ba27b511f37] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=71.7 K 2024-11-10T17:23:53,613 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a538739675843e9988e91c2cd4f2222, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259424903 2024-11-10T17:23:53,614 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e15179f0da9a43bdaf5102ff0b9b3ae8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731259424821 2024-11-10T17:23:53,614 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e940f5ba72974f0a9e9a868c6844d4ed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731259426098 2024-11-10T17:23:53,614 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 0928162543a949ef91bb0855c985e092, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259424903 2024-11-10T17:23:53,614 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting de83099b51c647a8ad49c1d55d423380, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731259426098 2024-11-10T17:23:53,614 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598d8272d03741eaad47572523aaf726, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731259427232 2024-11-10T17:23:53,614 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e2a310eb754077b9fe891d004d603f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731259429540 2024-11-10T17:23:53,614 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting da109ba9762c46bb916b5fe2e0a1877a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731259427232 2024-11-10T17:23:53,615 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e668525b516b4194b3210ccd76943d44, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731259431926 2024-11-10T17:23:53,615 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c8c64236d540f28c3330aa9f702549, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731259429540 2024-11-10T17:23:53,615 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting ebbab6c50c234b0baa6e2ba27b511f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731259431926 2024-11-10T17:23:53,624 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:53,628 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#B#compaction#352 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:53,629 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411101f250470a062481c9ec36da316647744_a43b207a293e67a9738e21c91e1dc09e store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:53,629 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/7d9bf2dd03044b5c9dde88cf23543c06 is 50, key is test_row_0/B:col10/1731259433134/Put/seqid=0 2024-11-10T17:23:53,632 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411101f250470a062481c9ec36da316647744_a43b207a293e67a9738e21c91e1dc09e, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:53,633 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101f250470a062481c9ec36da316647744_a43b207a293e67a9738e21c91e1dc09e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:53,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742245_1421 (size=12915) 2024-11-10T17:23:53,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742246_1422 (size=4469) 2024-11-10T17:23:53,637 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#A#compaction#351 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:53,638 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09 is 175, key is test_row_0/A:col10/1731259433134/Put/seqid=0 2024-11-10T17:23:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:53,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T17:23:53,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742247_1423 (size=31869) 2024-11-10T17:23:53,646 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09 2024-11-10T17:23:53,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100710d347e30f41158913f0e8dcf80950_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259433297/Put/seqid=0 2024-11-10T17:23:53,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,654 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/A of a43b207a293e67a9738e21c91e1dc09e into 676af7a9c3f94a2b9b3a6d0cd5ee3f09(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:53,654 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:53,654 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/A, priority=10, startTime=1731259433610; duration=0sec 2024-11-10T17:23:53,655 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:53,655 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:A 2024-11-10T17:23:53,655 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T17:23:53,656 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73466 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T17:23:53,656 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/C is initiating minor compaction (all files) 2024-11-10T17:23:53,656 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/C in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,656 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/dc2309d64cd34b58a81877bf37b42931, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e26b589170f5446c8db7eba5bf840387, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/70625b480cfa40779512dfc2e51dda7e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3f15d5ad00b44ac08bd247c9111f27a5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/2c838054ae3e4eedabb913959d5fcda8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/045a168222c044c785c58818c28e3898] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=71.7 K 2024-11-10T17:23:53,656 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc2309d64cd34b58a81877bf37b42931, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731259424821 2024-11-10T17:23:53,657 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e26b589170f5446c8db7eba5bf840387, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259424903 2024-11-10T17:23:53,657 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70625b480cfa40779512dfc2e51dda7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731259426098 2024-11-10T17:23:53,658 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f15d5ad00b44ac08bd247c9111f27a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731259427232 2024-11-10T17:23:53,658 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c838054ae3e4eedabb913959d5fcda8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731259429540 2024-11-10T17:23:53,659 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 045a168222c044c785c58818c28e3898, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731259431926 2024-11-10T17:23:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T17:23:53,667 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:53,667 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/bf360dad8ed846bd8bd4e90c24b7d629 is 50, key is test_row_0/C:col10/1731259433134/Put/seqid=0 2024-11-10T17:23:53,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742248_1424 (size=14994) 2024-11-10T17:23:53,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:53,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100710d347e30f41158913f0e8dcf80950_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100710d347e30f41158913f0e8dcf80950_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:53,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742249_1425 (size=12915) 2024-11-10T17:23:53,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/9664e07b7e544eb287dbdcb46cf4d5af, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:53,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/9664e07b7e544eb287dbdcb46cf4d5af is 175, key is test_row_0/A:col10/1731259433297/Put/seqid=0 2024-11-10T17:23:53,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742250_1426 (size=39949) 2024-11-10T17:23:53,691 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/bf360dad8ed846bd8bd4e90c24b7d629 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/bf360dad8ed846bd8bd4e90c24b7d629 2024-11-10T17:23:53,695 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/C of a43b207a293e67a9738e21c91e1dc09e into bf360dad8ed846bd8bd4e90c24b7d629(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:53,695 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:53,695 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/C, priority=10, startTime=1731259433610; duration=0sec 2024-11-10T17:23:53,695 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:53,695 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:C 2024-11-10T17:23:53,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259493714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259493715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259493723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:53,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:53,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259493832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259493832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259493841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:53,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:53,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:53,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:53,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T17:23:54,037 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/7d9bf2dd03044b5c9dde88cf23543c06 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/7d9bf2dd03044b5c9dde88cf23543c06 2024-11-10T17:23:54,041 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/B of a43b207a293e67a9738e21c91e1dc09e into 7d9bf2dd03044b5c9dde88cf23543c06(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:54,041 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:54,041 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/B, priority=10, startTime=1731259433610; duration=0sec 2024-11-10T17:23:54,041 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:54,041 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:B 2024-11-10T17:23:54,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259494042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259494042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259494061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/9664e07b7e544eb287dbdcb46cf4d5af 2024-11-10T17:23:54,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e0cde3faa9f84234aea7cb0134029a19 is 50, key is test_row_0/B:col10/1731259433297/Put/seqid=0 2024-11-10T17:23:54,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742251_1427 (size=12301) 2024-11-10T17:23:54,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e0cde3faa9f84234aea7cb0134029a19 2024-11-10T17:23:54,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:54,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/49d1311c64004bb1abfe5ad57d0313db is 50, key is test_row_0/C:col10/1731259433297/Put/seqid=0 2024-11-10T17:23:54,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742252_1428 (size=12301) 2024-11-10T17:23:54,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:54,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:54,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:54,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259494351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259494351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38648 deadline: 1731259494382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38634 deadline: 1731259494382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259494382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,392 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,392 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in a43b207a293e67a9738e21c91e1dc09e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:54,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:23:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T17:23:54,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/49d1311c64004bb1abfe5ad57d0313db 2024-11-10T17:23:54,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/9664e07b7e544eb287dbdcb46cf4d5af as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/9664e07b7e544eb287dbdcb46cf4d5af 2024-11-10T17:23:54,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/9664e07b7e544eb287dbdcb46cf4d5af, entries=200, sequenceid=287, filesize=39.0 K 2024-11-10T17:23:54,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e0cde3faa9f84234aea7cb0134029a19 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e0cde3faa9f84234aea7cb0134029a19 2024-11-10T17:23:54,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e0cde3faa9f84234aea7cb0134029a19, entries=150, sequenceid=287, filesize=12.0 K 2024-11-10T17:23:54,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/49d1311c64004bb1abfe5ad57d0313db as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/49d1311c64004bb1abfe5ad57d0313db 2024-11-10T17:23:54,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/49d1311c64004bb1abfe5ad57d0313db, entries=150, sequenceid=287, filesize=12.0 K 2024-11-10T17:23:54,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a43b207a293e67a9738e21c91e1dc09e in 911ms, sequenceid=287, compaction requested=false 2024-11-10T17:23:54,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:54,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T17:23:54,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:54,580 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T17:23:54,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111081a3c59c2bf94541970d23fe79caf202_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259433710/Put/seqid=0 2024-11-10T17:23:54,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742253_1429 (size=12454) 2024-11-10T17:23:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:54,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. as already flushing 2024-11-10T17:23:54,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259494915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259494921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:54,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259494928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:55,011 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111081a3c59c2bf94541970d23fe79caf202_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111081a3c59c2bf94541970d23fe79caf202_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/b5577bea28c6434abdeb5f1fc1b4d164, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/b5577bea28c6434abdeb5f1fc1b4d164 is 175, key is test_row_0/A:col10/1731259433710/Put/seqid=0 2024-11-10T17:23:55,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742254_1430 (size=31255) 2024-11-10T17:23:55,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259495032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259495033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259495041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259495251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259495251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259495251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,284 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-10T17:23:55,284 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:55,284 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,284 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:55,289 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-10T17:23:55,289 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:55,289 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,289 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:55,291 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-10T17:23:55,291 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:55,291 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,291 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:55,294 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-10T17:23:55,294 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:55,294 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,294 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:55,299 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-10T17:23:55,299 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:55,299 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,299 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:55,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:55,416 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/b5577bea28c6434abdeb5f1fc1b4d164 2024-11-10T17:23:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/dd5a8235b76b4f7d95af83818cd2082d is 50, key is test_row_0/B:col10/1731259433710/Put/seqid=0 2024-11-10T17:23:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742255_1431 (size=12301) 2024-11-10T17:23:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T17:23:55,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259495562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259495572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259495572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:55,826 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/dd5a8235b76b4f7d95af83818cd2082d 2024-11-10T17:23:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/d6abc173252d47b380047f1245fa2b35 is 50, key is test_row_0/C:col10/1731259433710/Put/seqid=0 2024-11-10T17:23:55,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742256_1432 (size=12301) 2024-11-10T17:23:56,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38680 deadline: 1731259496071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:56,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:56,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:23:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38664 deadline: 1731259496082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38656 deadline: 1731259496082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:23:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:23:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:23:56,237 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/d6abc173252d47b380047f1245fa2b35 2024-11-10T17:23:56,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/b5577bea28c6434abdeb5f1fc1b4d164 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/b5577bea28c6434abdeb5f1fc1b4d164 2024-11-10T17:23:56,243 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/b5577bea28c6434abdeb5f1fc1b4d164, entries=150, sequenceid=309, filesize=30.5 K 2024-11-10T17:23:56,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/dd5a8235b76b4f7d95af83818cd2082d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/dd5a8235b76b4f7d95af83818cd2082d 2024-11-10T17:23:56,246 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/dd5a8235b76b4f7d95af83818cd2082d, entries=150, sequenceid=309, filesize=12.0 K 2024-11-10T17:23:56,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/d6abc173252d47b380047f1245fa2b35 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/d6abc173252d47b380047f1245fa2b35 2024-11-10T17:23:56,250 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/d6abc173252d47b380047f1245fa2b35, entries=150, sequenceid=309, filesize=12.0 K 2024-11-10T17:23:56,250 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a43b207a293e67a9738e21c91e1dc09e in 1670ms, sequenceid=309, compaction requested=true 2024-11-10T17:23:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:56,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-10T17:23:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-10T17:23:56,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T17:23:56,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9070 sec 2024-11-10T17:23:56,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.9120 sec 2024-11-10T17:23:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:23:57,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T17:23:57,083 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-10T17:23:57,083 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:57,083 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:57,083 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:57,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:57,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cd46f21b5489407084dc18ab3b45e618_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259434919/Put/seqid=0 2024-11-10T17:23:57,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742257_1433 (size=12454) 2024-11-10T17:23:57,093 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-10T17:23:57,093 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-10T17:23:57,093 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:57,093 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:57,093 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:23:57,093 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:57,093 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:57,093 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:23:57,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:57,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:23:57,493 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:23:57,496 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cd46f21b5489407084dc18ab3b45e618_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd46f21b5489407084dc18ab3b45e618_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:23:57,497 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7adba517e26246399dbbfd7f5059af43, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:57,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7adba517e26246399dbbfd7f5059af43 is 175, key is test_row_0/A:col10/1731259434919/Put/seqid=0 2024-11-10T17:23:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T17:23:57,502 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:23:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742258_1434 (size=31255) 2024-11-10T17:23:57,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7adba517e26246399dbbfd7f5059af43 2024-11-10T17:23:57,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e82d463710c34310a8bf709d14aa1d1b is 50, key is test_row_0/B:col10/1731259434919/Put/seqid=0 2024-11-10T17:23:57,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742259_1435 (size=12301) 2024-11-10T17:23:58,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e82d463710c34310a8bf709d14aa1d1b 2024-11-10T17:23:58,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/812d8ff249be4d83af020a101ef1498b is 50, key is test_row_0/C:col10/1731259434919/Put/seqid=0 2024-11-10T17:23:58,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742260_1436 (size=12301) 2024-11-10T17:23:58,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/812d8ff249be4d83af020a101ef1498b 2024-11-10T17:23:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/7adba517e26246399dbbfd7f5059af43 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7adba517e26246399dbbfd7f5059af43 2024-11-10T17:23:58,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7adba517e26246399dbbfd7f5059af43, entries=150, sequenceid=327, filesize=30.5 K 2024-11-10T17:23:58,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/e82d463710c34310a8bf709d14aa1d1b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e82d463710c34310a8bf709d14aa1d1b 2024-11-10T17:23:58,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e82d463710c34310a8bf709d14aa1d1b, entries=150, sequenceid=327, filesize=12.0 K 2024-11-10T17:23:58,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/812d8ff249be4d83af020a101ef1498b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/812d8ff249be4d83af020a101ef1498b 2024-11-10T17:23:58,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/812d8ff249be4d83af020a101ef1498b, entries=150, sequenceid=327, filesize=12.0 K 2024-11-10T17:23:58,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for a43b207a293e67a9738e21c91e1dc09e in 1659ms, sequenceid=327, compaction requested=true 2024-11-10T17:23:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:23:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:23:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:58,743 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a43b207a293e67a9738e21c91e1dc09e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:23:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:58,743 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/B is initiating minor compaction (all files) 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:58,744 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/B in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/A is initiating minor compaction (all files) 2024-11-10T17:23:58,744 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/A in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:58,744 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/7d9bf2dd03044b5c9dde88cf23543c06, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e0cde3faa9f84234aea7cb0134029a19, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/dd5a8235b76b4f7d95af83818cd2082d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e82d463710c34310a8bf709d14aa1d1b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=48.7 K 2024-11-10T17:23:58,744 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/9664e07b7e544eb287dbdcb46cf4d5af, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/b5577bea28c6434abdeb5f1fc1b4d164, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7adba517e26246399dbbfd7f5059af43] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=131.2 K 2024-11-10T17:23:58,744 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/9664e07b7e544eb287dbdcb46cf4d5af, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/b5577bea28c6434abdeb5f1fc1b4d164, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7adba517e26246399dbbfd7f5059af43] 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d9bf2dd03044b5c9dde88cf23543c06, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731259431926 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 676af7a9c3f94a2b9b3a6d0cd5ee3f09, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731259431926 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e0cde3faa9f84234aea7cb0134029a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731259433284 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9664e07b7e544eb287dbdcb46cf4d5af, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731259433237 2024-11-10T17:23:58,744 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5a8235b76b4f7d95af83818cd2082d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731259433710 2024-11-10T17:23:58,745 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5577bea28c6434abdeb5f1fc1b4d164, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731259433710 2024-11-10T17:23:58,745 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e82d463710c34310a8bf709d14aa1d1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731259434919 2024-11-10T17:23:58,745 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7adba517e26246399dbbfd7f5059af43, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731259434919 2024-11-10T17:23:58,751 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:58,753 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:58,753 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110eb356b9f56674452aa16fe7ddce396d6_a43b207a293e67a9738e21c91e1dc09e store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:58,753 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/548b30abe36a455bbd9f084d69835d35 is 50, key is test_row_0/B:col10/1731259434919/Put/seqid=0 2024-11-10T17:23:58,758 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110eb356b9f56674452aa16fe7ddce396d6_a43b207a293e67a9738e21c91e1dc09e, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:58,758 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb356b9f56674452aa16fe7ddce396d6_a43b207a293e67a9738e21c91e1dc09e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:23:58,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742261_1437 (size=13051) 2024-11-10T17:23:58,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742262_1438 (size=4469) 2024-11-10T17:23:59,163 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#A#compaction#364 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:59,164 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/145f8048f1634ffdb206442840d25088 is 175, key is test_row_0/A:col10/1731259434919/Put/seqid=0 2024-11-10T17:23:59,165 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/548b30abe36a455bbd9f084d69835d35 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/548b30abe36a455bbd9f084d69835d35 2024-11-10T17:23:59,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742263_1439 (size=32005) 2024-11-10T17:23:59,169 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/B of a43b207a293e67a9738e21c91e1dc09e into 548b30abe36a455bbd9f084d69835d35(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:59,169 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:59,170 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/B, priority=12, startTime=1731259438743; duration=0sec 2024-11-10T17:23:59,170 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:23:59,170 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:B 2024-11-10T17:23:59,170 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:23:59,171 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:23:59,171 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): a43b207a293e67a9738e21c91e1dc09e/C is initiating minor compaction (all files) 2024-11-10T17:23:59,171 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a43b207a293e67a9738e21c91e1dc09e/C in TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:23:59,171 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/bf360dad8ed846bd8bd4e90c24b7d629, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/49d1311c64004bb1abfe5ad57d0313db, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/d6abc173252d47b380047f1245fa2b35, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/812d8ff249be4d83af020a101ef1498b] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp, totalSize=48.7 K 2024-11-10T17:23:59,171 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting bf360dad8ed846bd8bd4e90c24b7d629, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731259431926 2024-11-10T17:23:59,171 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 49d1311c64004bb1abfe5ad57d0313db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731259433284 2024-11-10T17:23:59,172 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d6abc173252d47b380047f1245fa2b35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731259433710 2024-11-10T17:23:59,172 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 812d8ff249be4d83af020a101ef1498b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731259434919 2024-11-10T17:23:59,179 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a43b207a293e67a9738e21c91e1dc09e#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:23:59,179 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3b4763ca6af84f5abbafbfbb59a76d8b is 50, key is test_row_0/C:col10/1731259434919/Put/seqid=0 2024-11-10T17:23:59,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742264_1440 (size=13051) 2024-11-10T17:23:59,573 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/145f8048f1634ffdb206442840d25088 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/145f8048f1634ffdb206442840d25088 2024-11-10T17:23:59,577 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/A of a43b207a293e67a9738e21c91e1dc09e into 145f8048f1634ffdb206442840d25088(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:59,577 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:59,577 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/A, priority=12, startTime=1731259438743; duration=0sec 2024-11-10T17:23:59,577 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:59,577 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:A 2024-11-10T17:23:59,586 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/3b4763ca6af84f5abbafbfbb59a76d8b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3b4763ca6af84f5abbafbfbb59a76d8b 2024-11-10T17:23:59,589 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a43b207a293e67a9738e21c91e1dc09e/C of a43b207a293e67a9738e21c91e1dc09e into 3b4763ca6af84f5abbafbfbb59a76d8b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:23:59,589 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a43b207a293e67a9738e21c91e1dc09e: 2024-11-10T17:23:59,590 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., storeName=a43b207a293e67a9738e21c91e1dc09e/C, priority=12, startTime=1731259438743; duration=0sec 2024-11-10T17:23:59,590 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:23:59,590 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a43b207a293e67a9738e21c91e1dc09e:C 2024-11-10T17:24:04,403 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-10T17:24:04,403 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:04,403 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:04,403 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:04,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:04,474 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-10T17:24:04,474 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:04,474 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:04,474 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:04,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1131 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3383 rows 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1134 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3398 rows 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1135 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3400 rows 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3387 rows 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1131 2024-11-10T17:24:04,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3391 rows 2024-11-10T17:24:04,475 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:24:04,475 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:24:04,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:04,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:04,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:04,476 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T17:24:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T17:24:04,482 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259444482"}]},"ts":"1731259444482"} 2024-11-10T17:24:04,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T17:24:04,484 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T17:24:04,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:24:04,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, UNASSIGN}] 2024-11-10T17:24:04,486 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, UNASSIGN 2024-11-10T17:24:04,487 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:04,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, UNASSIGN because future has completed 2024-11-10T17:24:04,489 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:24:04,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:24:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T17:24:04,641 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:04,641 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:24:04,641 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing a43b207a293e67a9738e21c91e1dc09e, disabling compactions & flushes 2024-11-10T17:24:04,641 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:24:04,641 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:24:04,641 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. after waiting 0 ms 2024-11-10T17:24:04,641 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:24:04,642 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing a43b207a293e67a9738e21c91e1dc09e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T17:24:04,647 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a43440289e894ff1b34dbd39e2b7b2c6_a43b207a293e67a9738e21c91e1dc09e is 50, key is test_row_0/A:col10/1731259437092/Put/seqid=0 2024-11-10T17:24:04,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742265_1441 (size=12454) 2024-11-10T17:24:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T17:24:04,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T17:24:05,051 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:05,055 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a43440289e894ff1b34dbd39e2b7b2c6_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a43440289e894ff1b34dbd39e2b7b2c6_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:05,056 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/261890d1bb1741b0a62c6b4c63c6d36e, store: [table=TestAcidGuarantees family=A region=a43b207a293e67a9738e21c91e1dc09e] 2024-11-10T17:24:05,057 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/261890d1bb1741b0a62c6b4c63c6d36e is 175, key is test_row_0/A:col10/1731259437092/Put/seqid=0 2024-11-10T17:24:05,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742266_1442 (size=31255) 2024-11-10T17:24:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T17:24:05,461 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/261890d1bb1741b0a62c6b4c63c6d36e 2024-11-10T17:24:05,468 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/0e7f55afe9204a7a9e0634e0dd55f72e is 50, key is test_row_0/B:col10/1731259437092/Put/seqid=0 2024-11-10T17:24:05,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742267_1443 (size=12301) 2024-11-10T17:24:05,471 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/0e7f55afe9204a7a9e0634e0dd55f72e 2024-11-10T17:24:05,477 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/cde2366c8dcd4b17867766d13a8f5169 is 50, key is test_row_0/C:col10/1731259437092/Put/seqid=0 2024-11-10T17:24:05,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742268_1444 (size=12301) 2024-11-10T17:24:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T17:24:05,880 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/cde2366c8dcd4b17867766d13a8f5169 2024-11-10T17:24:05,884 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/A/261890d1bb1741b0a62c6b4c63c6d36e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/261890d1bb1741b0a62c6b4c63c6d36e 2024-11-10T17:24:05,887 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/261890d1bb1741b0a62c6b4c63c6d36e, entries=150, sequenceid=337, filesize=30.5 K 2024-11-10T17:24:05,887 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/B/0e7f55afe9204a7a9e0634e0dd55f72e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0e7f55afe9204a7a9e0634e0dd55f72e 2024-11-10T17:24:05,891 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0e7f55afe9204a7a9e0634e0dd55f72e, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T17:24:05,892 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/.tmp/C/cde2366c8dcd4b17867766d13a8f5169 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/cde2366c8dcd4b17867766d13a8f5169 2024-11-10T17:24:05,895 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/cde2366c8dcd4b17867766d13a8f5169, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T17:24:05,896 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a43b207a293e67a9738e21c91e1dc09e in 1255ms, sequenceid=337, compaction requested=false 2024-11-10T17:24:05,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5279420d1b9b45cf936eba3da67405dd, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5e9a8894858c41be93fc2082ca16b2bf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/baa447ccc0984ce784729236b383b73f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/0c5920f753034bc3b269fdb257002c87, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7ecd10dc57cd465a84ae76d6bb5ad56f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/82b9b8af007a4e12841ddb0ce4ec3f89, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/59fb64847aa54bd198a6bd6bdf2ab187, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5c456c70b7c749988ead666df0b91b93, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/8912d8cfc66c497fa2292248a75956c0, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/4b8e680a69dc46d7b519754e4d04e4c4, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/1bdd58d6764e40b899746fccd7211017, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/6c14188104dd4152a9fb40485726f877, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/75c3e617597f43dca39c9397e54a5a18, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7a538739675843e9988e91c2cd4f2222, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e940f5ba72974f0a9e9a868c6844d4ed, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/598d8272d03741eaad47572523aaf726, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/46e2a310eb754077b9fe891d004d603f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e668525b516b4194b3210ccd76943d44, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/9664e07b7e544eb287dbdcb46cf4d5af, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/b5577bea28c6434abdeb5f1fc1b4d164, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7adba517e26246399dbbfd7f5059af43] to archive 2024-11-10T17:24:05,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:24:05,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5279420d1b9b45cf936eba3da67405dd to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5279420d1b9b45cf936eba3da67405dd 2024-11-10T17:24:05,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5e9a8894858c41be93fc2082ca16b2bf to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5e9a8894858c41be93fc2082ca16b2bf 2024-11-10T17:24:05,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/baa447ccc0984ce784729236b383b73f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/baa447ccc0984ce784729236b383b73f 2024-11-10T17:24:05,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/0c5920f753034bc3b269fdb257002c87 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/0c5920f753034bc3b269fdb257002c87 2024-11-10T17:24:05,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7ecd10dc57cd465a84ae76d6bb5ad56f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7ecd10dc57cd465a84ae76d6bb5ad56f 2024-11-10T17:24:05,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/82b9b8af007a4e12841ddb0ce4ec3f89 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/82b9b8af007a4e12841ddb0ce4ec3f89 2024-11-10T17:24:05,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/59fb64847aa54bd198a6bd6bdf2ab187 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/59fb64847aa54bd198a6bd6bdf2ab187 2024-11-10T17:24:05,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5c456c70b7c749988ead666df0b91b93 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/5c456c70b7c749988ead666df0b91b93 2024-11-10T17:24:05,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/8912d8cfc66c497fa2292248a75956c0 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/8912d8cfc66c497fa2292248a75956c0 2024-11-10T17:24:05,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/4b8e680a69dc46d7b519754e4d04e4c4 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/4b8e680a69dc46d7b519754e4d04e4c4 2024-11-10T17:24:05,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/1bdd58d6764e40b899746fccd7211017 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/1bdd58d6764e40b899746fccd7211017 2024-11-10T17:24:05,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/6c14188104dd4152a9fb40485726f877 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/6c14188104dd4152a9fb40485726f877 2024-11-10T17:24:05,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/75c3e617597f43dca39c9397e54a5a18 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/75c3e617597f43dca39c9397e54a5a18 2024-11-10T17:24:05,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7a538739675843e9988e91c2cd4f2222 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7a538739675843e9988e91c2cd4f2222 2024-11-10T17:24:05,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e940f5ba72974f0a9e9a868c6844d4ed to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e940f5ba72974f0a9e9a868c6844d4ed 2024-11-10T17:24:05,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/598d8272d03741eaad47572523aaf726 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/598d8272d03741eaad47572523aaf726 2024-11-10T17:24:05,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/46e2a310eb754077b9fe891d004d603f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/46e2a310eb754077b9fe891d004d603f 2024-11-10T17:24:05,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e668525b516b4194b3210ccd76943d44 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/e668525b516b4194b3210ccd76943d44 2024-11-10T17:24:05,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/676af7a9c3f94a2b9b3a6d0cd5ee3f09 2024-11-10T17:24:05,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/9664e07b7e544eb287dbdcb46cf4d5af to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/9664e07b7e544eb287dbdcb46cf4d5af 2024-11-10T17:24:05,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/b5577bea28c6434abdeb5f1fc1b4d164 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/b5577bea28c6434abdeb5f1fc1b4d164 2024-11-10T17:24:05,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7adba517e26246399dbbfd7f5059af43 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/7adba517e26246399dbbfd7f5059af43 2024-11-10T17:24:05,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b429b7d275524630be7c6925f8655996, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/f106948845654af5837335c253bf109e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/4066066c71b0498a820321f150c29058, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/a52d4b4bd1544e81928144667f7ddab8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/29ca2b5408e3482781eb4ab35c7f4ab2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b1f37e832076453881ec82af3652985f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e13381208e444137bb06c7d837fcd9ba, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/6878a83074534900a4c20a42f9d35c02, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/5fee10f8b89b4c1ab8f2389e1cf11b96, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/2b31bcdb5d0644cca7e5bc472b32daca, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d0422cec4bc54bf49721b57f238b6b31, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e15179f0da9a43bdaf5102ff0b9b3ae8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/751aac39039d4ee886070b84d7a75641, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0928162543a949ef91bb0855c985e092, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/de83099b51c647a8ad49c1d55d423380, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/da109ba9762c46bb916b5fe2e0a1877a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d3c8c64236d540f28c3330aa9f702549, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/7d9bf2dd03044b5c9dde88cf23543c06, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/ebbab6c50c234b0baa6e2ba27b511f37, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e0cde3faa9f84234aea7cb0134029a19, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/dd5a8235b76b4f7d95af83818cd2082d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e82d463710c34310a8bf709d14aa1d1b] to archive 2024-11-10T17:24:05,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:24:05,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b429b7d275524630be7c6925f8655996 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b429b7d275524630be7c6925f8655996 2024-11-10T17:24:05,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/f106948845654af5837335c253bf109e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/f106948845654af5837335c253bf109e 2024-11-10T17:24:05,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/4066066c71b0498a820321f150c29058 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/4066066c71b0498a820321f150c29058 2024-11-10T17:24:05,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/a52d4b4bd1544e81928144667f7ddab8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/a52d4b4bd1544e81928144667f7ddab8 2024-11-10T17:24:05,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/29ca2b5408e3482781eb4ab35c7f4ab2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/29ca2b5408e3482781eb4ab35c7f4ab2 2024-11-10T17:24:05,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b1f37e832076453881ec82af3652985f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/b1f37e832076453881ec82af3652985f 2024-11-10T17:24:05,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e13381208e444137bb06c7d837fcd9ba to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e13381208e444137bb06c7d837fcd9ba 2024-11-10T17:24:05,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/6878a83074534900a4c20a42f9d35c02 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/6878a83074534900a4c20a42f9d35c02 2024-11-10T17:24:05,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/5fee10f8b89b4c1ab8f2389e1cf11b96 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/5fee10f8b89b4c1ab8f2389e1cf11b96 2024-11-10T17:24:05,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/2b31bcdb5d0644cca7e5bc472b32daca to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/2b31bcdb5d0644cca7e5bc472b32daca 2024-11-10T17:24:05,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d0422cec4bc54bf49721b57f238b6b31 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d0422cec4bc54bf49721b57f238b6b31 2024-11-10T17:24:05,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e15179f0da9a43bdaf5102ff0b9b3ae8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e15179f0da9a43bdaf5102ff0b9b3ae8 2024-11-10T17:24:05,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/751aac39039d4ee886070b84d7a75641 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/751aac39039d4ee886070b84d7a75641 2024-11-10T17:24:05,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0928162543a949ef91bb0855c985e092 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0928162543a949ef91bb0855c985e092 2024-11-10T17:24:05,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/de83099b51c647a8ad49c1d55d423380 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/de83099b51c647a8ad49c1d55d423380 2024-11-10T17:24:05,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/da109ba9762c46bb916b5fe2e0a1877a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/da109ba9762c46bb916b5fe2e0a1877a 2024-11-10T17:24:05,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d3c8c64236d540f28c3330aa9f702549 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/d3c8c64236d540f28c3330aa9f702549 2024-11-10T17:24:05,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/7d9bf2dd03044b5c9dde88cf23543c06 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/7d9bf2dd03044b5c9dde88cf23543c06 2024-11-10T17:24:05,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/ebbab6c50c234b0baa6e2ba27b511f37 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/ebbab6c50c234b0baa6e2ba27b511f37 2024-11-10T17:24:05,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e0cde3faa9f84234aea7cb0134029a19 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e0cde3faa9f84234aea7cb0134029a19 2024-11-10T17:24:05,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/dd5a8235b76b4f7d95af83818cd2082d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/dd5a8235b76b4f7d95af83818cd2082d 2024-11-10T17:24:05,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e82d463710c34310a8bf709d14aa1d1b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/e82d463710c34310a8bf709d14aa1d1b 2024-11-10T17:24:05,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a97b6ad5018e4b70a22f2639ff55c167, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4c700afccd484879ae6ddfd91b2c0402, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4feffe30e35d4db195b655a5f3ecee4c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/292d6643cd4f402a97fe6610d23efdba, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/61ab69714ab64bfdbf0c892522a68bae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8fbebd65a422437585cde8ca79b4107b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8f9039535a9248d882f24f90918bffbc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3278eb4edc9840b6856b7ab175e336b5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a757f4d3ba2744cf94beefd28adb051d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e50ec84c56b9458499270de87888d832, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e5b4150aaf7e4b1c9c665a4648fa8409, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/dc2309d64cd34b58a81877bf37b42931, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a0f56aa9243f409696379826b85f767e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e26b589170f5446c8db7eba5bf840387, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/70625b480cfa40779512dfc2e51dda7e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3f15d5ad00b44ac08bd247c9111f27a5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/2c838054ae3e4eedabb913959d5fcda8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/bf360dad8ed846bd8bd4e90c24b7d629, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/045a168222c044c785c58818c28e3898, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/49d1311c64004bb1abfe5ad57d0313db, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/d6abc173252d47b380047f1245fa2b35, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/812d8ff249be4d83af020a101ef1498b] to archive 2024-11-10T17:24:05,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:24:05,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a97b6ad5018e4b70a22f2639ff55c167 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a97b6ad5018e4b70a22f2639ff55c167 2024-11-10T17:24:05,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4c700afccd484879ae6ddfd91b2c0402 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4c700afccd484879ae6ddfd91b2c0402 2024-11-10T17:24:05,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4feffe30e35d4db195b655a5f3ecee4c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/4feffe30e35d4db195b655a5f3ecee4c 2024-11-10T17:24:05,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/292d6643cd4f402a97fe6610d23efdba to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/292d6643cd4f402a97fe6610d23efdba 2024-11-10T17:24:05,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/61ab69714ab64bfdbf0c892522a68bae to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/61ab69714ab64bfdbf0c892522a68bae 2024-11-10T17:24:05,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8fbebd65a422437585cde8ca79b4107b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8fbebd65a422437585cde8ca79b4107b 2024-11-10T17:24:05,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8f9039535a9248d882f24f90918bffbc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/8f9039535a9248d882f24f90918bffbc 2024-11-10T17:24:05,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3278eb4edc9840b6856b7ab175e336b5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3278eb4edc9840b6856b7ab175e336b5 2024-11-10T17:24:05,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a757f4d3ba2744cf94beefd28adb051d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a757f4d3ba2744cf94beefd28adb051d 2024-11-10T17:24:05,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e50ec84c56b9458499270de87888d832 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e50ec84c56b9458499270de87888d832 2024-11-10T17:24:05,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e5b4150aaf7e4b1c9c665a4648fa8409 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e5b4150aaf7e4b1c9c665a4648fa8409 2024-11-10T17:24:05,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/dc2309d64cd34b58a81877bf37b42931 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/dc2309d64cd34b58a81877bf37b42931 2024-11-10T17:24:05,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a0f56aa9243f409696379826b85f767e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/a0f56aa9243f409696379826b85f767e 2024-11-10T17:24:05,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e26b589170f5446c8db7eba5bf840387 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/e26b589170f5446c8db7eba5bf840387 2024-11-10T17:24:05,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/70625b480cfa40779512dfc2e51dda7e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/70625b480cfa40779512dfc2e51dda7e 2024-11-10T17:24:05,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3f15d5ad00b44ac08bd247c9111f27a5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3f15d5ad00b44ac08bd247c9111f27a5 2024-11-10T17:24:05,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/2c838054ae3e4eedabb913959d5fcda8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/2c838054ae3e4eedabb913959d5fcda8 2024-11-10T17:24:05,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/bf360dad8ed846bd8bd4e90c24b7d629 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/bf360dad8ed846bd8bd4e90c24b7d629 2024-11-10T17:24:05,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/045a168222c044c785c58818c28e3898 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/045a168222c044c785c58818c28e3898 2024-11-10T17:24:05,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/49d1311c64004bb1abfe5ad57d0313db to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/49d1311c64004bb1abfe5ad57d0313db 2024-11-10T17:24:05,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/d6abc173252d47b380047f1245fa2b35 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/d6abc173252d47b380047f1245fa2b35 2024-11-10T17:24:05,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/812d8ff249be4d83af020a101ef1498b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/812d8ff249be4d83af020a101ef1498b 2024-11-10T17:24:05,968 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/recovered.edits/340.seqid, newMaxSeqId=340, maxSeqId=4 2024-11-10T17:24:05,968 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. 2024-11-10T17:24:05,968 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for a43b207a293e67a9738e21c91e1dc09e: Waiting for close lock at 1731259444641Running coprocessor pre-close hooks at 1731259444641Disabling compacts and flushes for region at 1731259444641Disabling writes for close at 1731259444641Obtaining lock to block concurrent updates at 1731259444642 (+1 ms)Preparing flush snapshotting stores in a43b207a293e67a9738e21c91e1dc09e at 1731259444642Finished memstore snapshotting TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731259444642Flushing stores of TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e. at 1731259444642Flushing a43b207a293e67a9738e21c91e1dc09e/A: creating writer at 1731259444642Flushing mob file a43b207a293e67a9738e21c91e1dc09e/A: appending metadata at 1731259444647 (+5 ms)Flushing mob file a43b207a293e67a9738e21c91e1dc09e/A: closing flushed file at 1731259444647Flushing a43b207a293e67a9738e21c91e1dc09e/A: appending metadata at 1731259445056 (+409 ms)Flushing a43b207a293e67a9738e21c91e1dc09e/A: closing flushed file at 1731259445056Flushing a43b207a293e67a9738e21c91e1dc09e/B: creating writer at 1731259445464 (+408 ms)Flushing a43b207a293e67a9738e21c91e1dc09e/B: appending metadata at 1731259445467 (+3 ms)Flushing a43b207a293e67a9738e21c91e1dc09e/B: closing flushed file at 1731259445467Flushing a43b207a293e67a9738e21c91e1dc09e/C: creating writer at 1731259445474 (+7 ms)Flushing a43b207a293e67a9738e21c91e1dc09e/C: appending metadata at 1731259445476 (+2 ms)Flushing a43b207a293e67a9738e21c91e1dc09e/C: closing flushed file at 1731259445476Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3643b1c: reopening flushed file at 1731259445883 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@724be6e7: reopening flushed file at 1731259445887 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e90ed0e: reopening flushed file at 1731259445891 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a43b207a293e67a9738e21c91e1dc09e in 1255ms, sequenceid=337, compaction requested=false at 1731259445896 (+5 ms)Writing region close event to WAL at 1731259445965 (+69 ms)Running coprocessor post-close hooks at 1731259445968 (+3 ms)Closed at 1731259445968 2024-11-10T17:24:05,970 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:05,971 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=a43b207a293e67a9738e21c91e1dc09e, regionState=CLOSED 2024-11-10T17:24:05,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:24:05,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T17:24:05,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure a43b207a293e67a9738e21c91e1dc09e, server=8e3de9160f9f,36421,1731259326553 in 1.4840 sec 2024-11-10T17:24:05,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-10T17:24:05,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a43b207a293e67a9738e21c91e1dc09e, UNASSIGN in 1.4890 sec 2024-11-10T17:24:05,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T17:24:05,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4930 sec 2024-11-10T17:24:05,981 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259445980"}]},"ts":"1731259445980"} 2024-11-10T17:24:05,982 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T17:24:05,982 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T17:24:05,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5070 sec 2024-11-10T17:24:06,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T17:24:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T17:24:06,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:06,632 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T17:24:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:06,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T17:24:06,636 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:06,637 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,640 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/recovered.edits] 2024-11-10T17:24:06,642 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/145f8048f1634ffdb206442840d25088 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/145f8048f1634ffdb206442840d25088 2024-11-10T17:24:06,644 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/261890d1bb1741b0a62c6b4c63c6d36e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/A/261890d1bb1741b0a62c6b4c63c6d36e 2024-11-10T17:24:06,646 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0e7f55afe9204a7a9e0634e0dd55f72e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/0e7f55afe9204a7a9e0634e0dd55f72e 2024-11-10T17:24:06,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/548b30abe36a455bbd9f084d69835d35 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/B/548b30abe36a455bbd9f084d69835d35 2024-11-10T17:24:06,649 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3b4763ca6af84f5abbafbfbb59a76d8b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/3b4763ca6af84f5abbafbfbb59a76d8b 2024-11-10T17:24:06,650 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/cde2366c8dcd4b17867766d13a8f5169 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/C/cde2366c8dcd4b17867766d13a8f5169 2024-11-10T17:24:06,653 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/recovered.edits/340.seqid to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e/recovered.edits/340.seqid 2024-11-10T17:24:06,653 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,653 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T17:24:06,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T17:24:06,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T17:24:06,657 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100710d347e30f41158913f0e8dcf80950_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100710d347e30f41158913f0e8dcf80950_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,658 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110149a9e263b974df4961ead3fcbf9bd75_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110149a9e263b974df4961ead3fcbf9bd75_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,659 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103af5d07958f94dfaaf94aedcccaf3f56_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103af5d07958f94dfaaf94aedcccaf3f56_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,660 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c6b05db470842268fdeb6994716a270_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c6b05db470842268fdeb6994716a270_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,661 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104dfcb337490b4029a38caade4a1c927e_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104dfcb337490b4029a38caade4a1c927e_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,662 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110536ca255e4ef4930a56e498988e2e79d_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110536ca255e4ef4930a56e498988e2e79d_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,663 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111057d344464e154c7784cc3c2ebbc69f07_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111057d344464e154c7784cc3c2ebbc69f07_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,664 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110688693ea4f1847eb87d763abf6174d1e_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110688693ea4f1847eb87d763abf6174d1e_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,666 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110769e3e49d2b247ec9835df2e1f8018e0_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110769e3e49d2b247ec9835df2e1f8018e0_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,667 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b87c69d9f4e4b0ab446fb8c56de61b6_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b87c69d9f4e4b0ab446fb8c56de61b6_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,668 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111081a3c59c2bf94541970d23fe79caf202_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111081a3c59c2bf94541970d23fe79caf202_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,670 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111085db11d11e934d74891635c8851a9b64_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111085db11d11e934d74891635c8851a9b64_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,671 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a43440289e894ff1b34dbd39e2b7b2c6_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a43440289e894ff1b34dbd39e2b7b2c6_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,672 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab83ed0890f94897b78cc633eb1dcf69_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab83ed0890f94897b78cc633eb1dcf69_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,673 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110acfdfc576fb14cba94b0a13ae8ddd1a9_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110acfdfc576fb14cba94b0a13ae8ddd1a9_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,675 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1cc64221f0e4291a653b28134cf1416_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1cc64221f0e4291a653b28134cf1416_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,676 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd46f21b5489407084dc18ab3b45e618_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd46f21b5489407084dc18ab3b45e618_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,678 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d6ce60277feb4e71af4db04ed98d852c_a43b207a293e67a9738e21c91e1dc09e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d6ce60277feb4e71af4db04ed98d852c_a43b207a293e67a9738e21c91e1dc09e 2024-11-10T17:24:06,680 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T17:24:06,682 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:06,685 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T17:24:06,687 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T17:24:06,689 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:06,689 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T17:24:06,689 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731259446689"}]},"ts":"9223372036854775807"} 2024-11-10T17:24:06,691 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T17:24:06,691 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a43b207a293e67a9738e21c91e1dc09e, NAME => 'TestAcidGuarantees,,1731259412180.a43b207a293e67a9738e21c91e1dc09e.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T17:24:06,691 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T17:24:06,692 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731259446691"}]},"ts":"9223372036854775807"} 2024-11-10T17:24:06,694 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T17:24:06,695 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:06,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-10T17:24:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T17:24:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T17:24:06,742 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:06,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 242), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=6031 (was 6122) 2024-11-10T17:24:06,767 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=6030 2024-11-10T17:24:06,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:24:06,769 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:24:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:06,771 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T17:24:06,772 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:06,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-10T17:24:06,773 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T17:24:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T17:24:06,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742269_1445 (size=959) 2024-11-10T17:24:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T17:24:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T17:24:07,181 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:24:07,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742270_1446 (size=53) 2024-11-10T17:24:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T17:24:07,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:24:07,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 57c77250ebf57e5a251700719e1bd916, disabling compactions & flushes 2024-11-10T17:24:07,586 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:07,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:07,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. after waiting 0 ms 2024-11-10T17:24:07,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:07,586 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:07,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 57c77250ebf57e5a251700719e1bd916: Waiting for close lock at 1731259447586Disabling compacts and flushes for region at 1731259447586Disabling writes for close at 1731259447586Writing region close event to WAL at 1731259447586Closed at 1731259447586 2024-11-10T17:24:07,587 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T17:24:07,587 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731259447587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731259447587"}]},"ts":"1731259447587"} 2024-11-10T17:24:07,589 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T17:24:07,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T17:24:07,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259447589"}]},"ts":"1731259447589"} 2024-11-10T17:24:07,590 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T17:24:07,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, ASSIGN}] 2024-11-10T17:24:07,591 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, ASSIGN 2024-11-10T17:24:07,592 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, ASSIGN; state=OFFLINE, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=false 2024-11-10T17:24:07,743 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=57c77250ebf57e5a251700719e1bd916, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:07,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, ASSIGN because future has completed 2024-11-10T17:24:07,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:24:07,899 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:07,899 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:24:07,900 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,900 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:24:07,900 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,900 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,901 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,902 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57c77250ebf57e5a251700719e1bd916 columnFamilyName A 2024-11-10T17:24:07,902 DEBUG [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:07,903 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] regionserver.HStore(327): Store=57c77250ebf57e5a251700719e1bd916/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:07,903 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,904 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57c77250ebf57e5a251700719e1bd916 columnFamilyName B 2024-11-10T17:24:07,904 DEBUG [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:07,904 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] regionserver.HStore(327): Store=57c77250ebf57e5a251700719e1bd916/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:07,904 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,905 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57c77250ebf57e5a251700719e1bd916 columnFamilyName C 2024-11-10T17:24:07,905 DEBUG [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:07,905 INFO [StoreOpener-57c77250ebf57e5a251700719e1bd916-1 {}] regionserver.HStore(327): Store=57c77250ebf57e5a251700719e1bd916/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:07,905 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,906 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,906 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,907 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,907 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,907 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:24:07,908 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,909 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:24:07,910 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 57c77250ebf57e5a251700719e1bd916; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75177401, jitterRate=0.12023057043552399}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:24:07,910 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:07,910 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 57c77250ebf57e5a251700719e1bd916: Running coprocessor pre-open hook at 1731259447900Writing region info on filesystem at 1731259447900Initializing all the Stores at 1731259447901 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259447901Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259447901Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259447901Cleaning up temporary data from old regions at 1731259447907 (+6 ms)Running coprocessor post-open hooks at 1731259447910 (+3 ms)Region opened successfully at 1731259447910 2024-11-10T17:24:07,911 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., pid=108, masterSystemTime=1731259447896 2024-11-10T17:24:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T17:24:07,913 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:07,913 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:07,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=57c77250ebf57e5a251700719e1bd916, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:07,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:24:07,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T17:24:07,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure 57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 in 171 msec 2024-11-10T17:24:07,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-10T17:24:07,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, ASSIGN in 326 msec 2024-11-10T17:24:07,918 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T17:24:07,918 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259447918"}]},"ts":"1731259447918"} 2024-11-10T17:24:07,920 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T17:24:07,920 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T17:24:07,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-10T17:24:08,316 WARN [regionserver/8e3de9160f9f:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-10T17:24:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T17:24:08,921 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:08,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,924 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c097b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57dd5a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,926 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,928 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T17:24:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,40007,1731259325831 2024-11-10T17:24:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53b647ab 2024-11-10T17:24:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T17:24:08,930 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T17:24:08,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,931 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6e224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,933 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716e75f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,934 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a319cd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3435272b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,937 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a983a2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,937 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27270632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,939 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5920923a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,940 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26980835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca5745d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,942 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5bbc4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c660fd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dcbe8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384beafa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b2001e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c889282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@285bea13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,952 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436210c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,953 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59eb8815, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:08,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:08,955 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5366812c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:08,956 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad48ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:08,957 DEBUG [Thread-2003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,957 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,957 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,958 DEBUG [Thread-2012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,958 DEBUG [Thread-2018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,958 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,958 DEBUG [Thread-2014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,958 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,959 DEBUG [Thread-2016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,959 DEBUG [Thread-2014 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,960 DEBUG [Thread-2014 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,960 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,961 DEBUG [Thread-1999 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-10T17:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T17:24:08,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:08,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,966 DEBUG [Thread-2010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,968 DEBUG [Thread-2010 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:08,968 DEBUG [Thread-2010 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:08,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2] 2024-11-10T17:24:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:08,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:08,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:24:09,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/08f0366d5ad140a495fc4f0cd4478488 is 50, key is test_row_0/A:col10/1731259448973/Put/seqid=0 2024-11-10T17:24:09,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742271_1447 (size=11997) 2024-11-10T17:24:09,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/08f0366d5ad140a495fc4f0cd4478488 2024-11-10T17:24:09,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259509006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259509008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259509009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259509012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259509013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/77cbf86083ab4376b70b0831f308bfa5 is 50, key is test_row_0/B:col10/1731259448973/Put/seqid=0 2024-11-10T17:24:09,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742272_1448 (size=9657) 2024-11-10T17:24:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T17:24:09,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T17:24:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259509122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259509131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259509131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259509131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259509131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T17:24:09,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:09,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T17:24:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259509331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259509341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259509342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259509342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259509352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T17:24:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/77cbf86083ab4376b70b0831f308bfa5 2024-11-10T17:24:09,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/e4c1581460204a50a927986ff7b74d4e is 50, key is test_row_0/C:col10/1731259448973/Put/seqid=0 2024-11-10T17:24:09,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742273_1449 (size=9657) 2024-11-10T17:24:09,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T17:24:09,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:09,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T17:24:09,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259509641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259509651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259509662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259509662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259509671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:09,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:09,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:09,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T17:24:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T17:24:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:09,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/e4c1581460204a50a927986ff7b74d4e 2024-11-10T17:24:09,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:09,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/08f0366d5ad140a495fc4f0cd4478488 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/08f0366d5ad140a495fc4f0cd4478488 2024-11-10T17:24:09,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/08f0366d5ad140a495fc4f0cd4478488, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T17:24:09,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/77cbf86083ab4376b70b0831f308bfa5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/77cbf86083ab4376b70b0831f308bfa5 2024-11-10T17:24:09,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/77cbf86083ab4376b70b0831f308bfa5, entries=100, sequenceid=14, filesize=9.4 K 2024-11-10T17:24:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/e4c1581460204a50a927986ff7b74d4e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/e4c1581460204a50a927986ff7b74d4e 2024-11-10T17:24:09,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/e4c1581460204a50a927986ff7b74d4e, entries=100, sequenceid=14, filesize=9.4 K 2024-11-10T17:24:09,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 57c77250ebf57e5a251700719e1bd916 in 945ms, sequenceid=14, compaction requested=false 2024-11-10T17:24:09,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:10,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T17:24:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:10,046 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T17:24:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/754cbc31788f4b3b96df59f1b4ae55e6 is 50, key is test_row_0/A:col10/1731259449011/Put/seqid=0 2024-11-10T17:24:10,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742274_1450 (size=12001) 2024-11-10T17:24:10,057 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/754cbc31788f4b3b96df59f1b4ae55e6 2024-11-10T17:24:10,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/7731cb58800840b390095ee5193b2630 is 50, key is test_row_0/B:col10/1731259449011/Put/seqid=0 2024-11-10T17:24:10,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742275_1451 (size=12001) 2024-11-10T17:24:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T17:24:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:10,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:10,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259510182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259510184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259510191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259510191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259510192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259510312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259510312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259510312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,471 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/7731cb58800840b390095ee5193b2630 2024-11-10T17:24:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/89bc591a19fd4c1fa3fe97106699db11 is 50, key is test_row_0/C:col10/1731259449011/Put/seqid=0 2024-11-10T17:24:10,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742276_1452 (size=12001) 2024-11-10T17:24:10,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259510532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259510532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259510532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259510841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259510841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259510841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:10,883 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/89bc591a19fd4c1fa3fe97106699db11 2024-11-10T17:24:10,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/754cbc31788f4b3b96df59f1b4ae55e6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/754cbc31788f4b3b96df59f1b4ae55e6 2024-11-10T17:24:10,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/754cbc31788f4b3b96df59f1b4ae55e6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T17:24:10,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/7731cb58800840b390095ee5193b2630 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7731cb58800840b390095ee5193b2630 2024-11-10T17:24:10,895 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7731cb58800840b390095ee5193b2630, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T17:24:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/89bc591a19fd4c1fa3fe97106699db11 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/89bc591a19fd4c1fa3fe97106699db11 2024-11-10T17:24:10,901 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/89bc591a19fd4c1fa3fe97106699db11, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T17:24:10,902 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 57c77250ebf57e5a251700719e1bd916 in 856ms, sequenceid=37, compaction requested=false 2024-11-10T17:24:10,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:10,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:10,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-10T17:24:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-10T17:24:10,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T17:24:10,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-11-10T17:24:10,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9460 sec 2024-11-10T17:24:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T17:24:11,112 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:11,113 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-10T17:24:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T17:24:11,114 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:11,115 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:11,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:11,207 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:11,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T17:24:11,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/6febe9343eb14a41af383e4dca916a49 is 50, key is test_row_0/A:col10/1731259451205/Put/seqid=0 2024-11-10T17:24:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T17:24:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742277_1453 (size=14341) 2024-11-10T17:24:11,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:11,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:11,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259511291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259511293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259511361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259511361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259511362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259511412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259511412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T17:24:11,490 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T17:24:11,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:11,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:11,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/6febe9343eb14a41af383e4dca916a49 2024-11-10T17:24:11,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/d372bee0e78a416f92944a8a668b7b0f is 50, key is test_row_0/B:col10/1731259451205/Put/seqid=0 2024-11-10T17:24:11,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259511632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259511632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742278_1454 (size=12001) 2024-11-10T17:24:11,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T17:24:11,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:11,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:11,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:11,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259511942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:11,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259511952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:11,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:11,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:12,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:12,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/d372bee0e78a416f92944a8a668b7b0f 2024-11-10T17:24:12,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/6f8c166d79854809a21c0e0cde1ca46c is 50, key is test_row_0/C:col10/1731259451205/Put/seqid=0 2024-11-10T17:24:12,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742279_1455 (size=12001) 2024-11-10T17:24:12,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T17:24:12,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259512381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259512382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:12,396 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,397 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259512395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:12,399 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/6f8c166d79854809a21c0e0cde1ca46c 2024-11-10T17:24:12,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/6febe9343eb14a41af383e4dca916a49 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6febe9343eb14a41af383e4dca916a49 2024-11-10T17:24:12,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6febe9343eb14a41af383e4dca916a49, entries=200, sequenceid=51, filesize=14.0 K 2024-11-10T17:24:12,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259512461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/d372bee0e78a416f92944a8a668b7b0f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d372bee0e78a416f92944a8a668b7b0f 2024-11-10T17:24:12,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d372bee0e78a416f92944a8a668b7b0f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T17:24:12,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/6f8c166d79854809a21c0e0cde1ca46c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6f8c166d79854809a21c0e0cde1ca46c 2024-11-10T17:24:12,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6f8c166d79854809a21c0e0cde1ca46c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T17:24:12,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 57c77250ebf57e5a251700719e1bd916 in 1270ms, sequenceid=51, compaction requested=true 2024-11-10T17:24:12,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:12,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:12,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:12,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:12,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:12,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:12,477 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:12,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:12,477 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:12,478 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:12,478 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:12,478 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:12,478 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:12,478 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,478 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,478 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/08f0366d5ad140a495fc4f0cd4478488, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/754cbc31788f4b3b96df59f1b4ae55e6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6febe9343eb14a41af383e4dca916a49] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=37.4 K 2024-11-10T17:24:12,478 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/77cbf86083ab4376b70b0831f308bfa5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7731cb58800840b390095ee5193b2630, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d372bee0e78a416f92944a8a668b7b0f] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=32.9 K 2024-11-10T17:24:12,479 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 77cbf86083ab4376b70b0831f308bfa5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731259448973 2024-11-10T17:24:12,479 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08f0366d5ad140a495fc4f0cd4478488, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731259448970 2024-11-10T17:24:12,479 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7731cb58800840b390095ee5193b2630, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731259449006 2024-11-10T17:24:12,479 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 754cbc31788f4b3b96df59f1b4ae55e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731259449006 2024-11-10T17:24:12,479 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d372bee0e78a416f92944a8a668b7b0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259450177 2024-11-10T17:24:12,479 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6febe9343eb14a41af383e4dca916a49, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259450177 2024-11-10T17:24:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:12,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T17:24:12,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/7fba275a67c540b7a6dd76372daf5985 is 50, key is test_row_0/A:col10/1731259451291/Put/seqid=0 2024-11-10T17:24:12,489 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:12,490 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5dce24b2ca6f4fa7b71a29db2aab604b is 50, key is test_row_0/A:col10/1731259451205/Put/seqid=0 2024-11-10T17:24:12,491 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:12,492 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/dd65ea156f984512a011529177db474b is 50, key is test_row_0/B:col10/1731259451205/Put/seqid=0 2024-11-10T17:24:12,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:12,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742281_1457 (size=12104) 2024-11-10T17:24:12,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742282_1458 (size=12104) 2024-11-10T17:24:12,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742280_1456 (size=14341) 2024-11-10T17:24:12,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/7fba275a67c540b7a6dd76372daf5985 2024-11-10T17:24:12,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/6ee48826f4ef4bd9937cd2cc6a7c6876 is 50, key is test_row_0/B:col10/1731259451291/Put/seqid=0 2024-11-10T17:24:12,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742283_1459 (size=12001) 2024-11-10T17:24:12,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/6ee48826f4ef4bd9937cd2cc6a7c6876 2024-11-10T17:24:12,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/bef92cda2f83419bbac5b65251fc7792 is 50, key is test_row_0/C:col10/1731259451291/Put/seqid=0 2024-11-10T17:24:12,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742284_1460 (size=12001) 2024-11-10T17:24:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259512570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:12,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:12,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259512691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,917 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/dd65ea156f984512a011529177db474b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/dd65ea156f984512a011529177db474b 2024-11-10T17:24:12,917 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5dce24b2ca6f4fa7b71a29db2aab604b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5dce24b2ca6f4fa7b71a29db2aab604b 2024-11-10T17:24:12,922 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into 5dce24b2ca6f4fa7b71a29db2aab604b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:12,922 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:12,922 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=13, startTime=1731259452477; duration=0sec 2024-11-10T17:24:12,922 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:12,922 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:12,922 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259512912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:12,924 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into dd65ea156f984512a011529177db474b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:12,924 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:12,924 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=13, startTime=1731259452477; duration=0sec 2024-11-10T17:24:12,924 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:12,924 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:12,925 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:12,925 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:12,925 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,925 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/e4c1581460204a50a927986ff7b74d4e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/89bc591a19fd4c1fa3fe97106699db11, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6f8c166d79854809a21c0e0cde1ca46c] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=32.9 K 2024-11-10T17:24:12,925 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c1581460204a50a927986ff7b74d4e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731259448973 2024-11-10T17:24:12,925 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89bc591a19fd4c1fa3fe97106699db11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731259449006 2024-11-10T17:24:12,926 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f8c166d79854809a21c0e0cde1ca46c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259450177 2024-11-10T17:24:12,936 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#383 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:12,937 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/81aff07caf2a4bf3ba94b2702866c8e6 is 50, key is test_row_0/C:col10/1731259451205/Put/seqid=0 2024-11-10T17:24:12,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742285_1461 (size=12104) 2024-11-10T17:24:12,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/bef92cda2f83419bbac5b65251fc7792 2024-11-10T17:24:12,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:12,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:12,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/7fba275a67c540b7a6dd76372daf5985 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7fba275a67c540b7a6dd76372daf5985 2024-11-10T17:24:12,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7fba275a67c540b7a6dd76372daf5985, entries=200, sequenceid=74, filesize=14.0 K 2024-11-10T17:24:12,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/6ee48826f4ef4bd9937cd2cc6a7c6876 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/6ee48826f4ef4bd9937cd2cc6a7c6876 2024-11-10T17:24:12,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/6ee48826f4ef4bd9937cd2cc6a7c6876, entries=150, sequenceid=74, filesize=11.7 K 2024-11-10T17:24:12,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/bef92cda2f83419bbac5b65251fc7792 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/bef92cda2f83419bbac5b65251fc7792 2024-11-10T17:24:12,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/bef92cda2f83419bbac5b65251fc7792, entries=150, sequenceid=74, filesize=11.7 K 2024-11-10T17:24:12,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 57c77250ebf57e5a251700719e1bd916 in 492ms, sequenceid=74, compaction requested=false 2024-11-10T17:24:12,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:13,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T17:24:13,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:13,109 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T17:24:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/7f0e5c293fd64a6c9ad90146fdca597a is 50, key is test_row_0/A:col10/1731259452536/Put/seqid=0 2024-11-10T17:24:13,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742286_1462 (size=12001) 2024-11-10T17:24:13,119 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/7f0e5c293fd64a6c9ad90146fdca597a 2024-11-10T17:24:13,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/c28eb65e99024920b87c136db67565b6 is 50, key is test_row_0/B:col10/1731259452536/Put/seqid=0 2024-11-10T17:24:13,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742287_1463 (size=12001) 2024-11-10T17:24:13,132 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/c28eb65e99024920b87c136db67565b6 2024-11-10T17:24:13,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/114f476687374a7e924d1e1ee4702c71 is 50, key is test_row_0/C:col10/1731259452536/Put/seqid=0 2024-11-10T17:24:13,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742288_1464 (size=12001) 2024-11-10T17:24:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:13,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T17:24:13,347 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/81aff07caf2a4bf3ba94b2702866c8e6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/81aff07caf2a4bf3ba94b2702866c8e6 2024-11-10T17:24:13,351 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into 81aff07caf2a4bf3ba94b2702866c8e6(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:13,351 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:13,351 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=13, startTime=1731259452477; duration=0sec 2024-11-10T17:24:13,352 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:13,352 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:13,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259513410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:13,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259513481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:13,489 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:13,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259513522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:13,545 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/114f476687374a7e924d1e1ee4702c71 2024-11-10T17:24:13,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/7f0e5c293fd64a6c9ad90146fdca597a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7f0e5c293fd64a6c9ad90146fdca597a 2024-11-10T17:24:13,552 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7f0e5c293fd64a6c9ad90146fdca597a, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T17:24:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/c28eb65e99024920b87c136db67565b6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c28eb65e99024920b87c136db67565b6 2024-11-10T17:24:13,555 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c28eb65e99024920b87c136db67565b6, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T17:24:13,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/114f476687374a7e924d1e1ee4702c71 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/114f476687374a7e924d1e1ee4702c71 2024-11-10T17:24:13,558 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/114f476687374a7e924d1e1ee4702c71, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T17:24:13,559 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 57c77250ebf57e5a251700719e1bd916 in 450ms, sequenceid=89, compaction requested=true 2024-11-10T17:24:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-10T17:24:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-10T17:24:13,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T17:24:13,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4460 sec 2024-11-10T17:24:13,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.4500 sec 2024-11-10T17:24:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:13,733 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:13,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:24:13,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/49a844ddcf0540af8af24bd46040eed0 is 50, key is test_row_0/A:col10/1731259453732/Put/seqid=0 2024-11-10T17:24:13,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742289_1465 (size=14341) 2024-11-10T17:24:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259513800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:13,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259513912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:14,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259514131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:14,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/49a844ddcf0540af8af24bd46040eed0 2024-11-10T17:24:14,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/51ceb7c149a341e3ad68368b6ff24cd7 is 50, key is test_row_0/B:col10/1731259453732/Put/seqid=0 2024-11-10T17:24:14,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742290_1466 (size=12001) 2024-11-10T17:24:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259514401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:14,409 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:14,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259514411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:14,421 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259514422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:14,430 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:14,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259514442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:14,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/51ceb7c149a341e3ad68368b6ff24cd7 2024-11-10T17:24:14,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/1ef4189c77fb4cf6b3c6be0ec13f5d52 is 50, key is test_row_0/C:col10/1731259453732/Put/seqid=0 2024-11-10T17:24:14,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742291_1467 (size=12001) 2024-11-10T17:24:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259514951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:14,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/1ef4189c77fb4cf6b3c6be0ec13f5d52 2024-11-10T17:24:14,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/49a844ddcf0540af8af24bd46040eed0 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/49a844ddcf0540af8af24bd46040eed0 2024-11-10T17:24:14,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/49a844ddcf0540af8af24bd46040eed0, entries=200, sequenceid=114, filesize=14.0 K 2024-11-10T17:24:14,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/51ceb7c149a341e3ad68368b6ff24cd7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/51ceb7c149a341e3ad68368b6ff24cd7 2024-11-10T17:24:14,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/51ceb7c149a341e3ad68368b6ff24cd7, entries=150, sequenceid=114, filesize=11.7 K 2024-11-10T17:24:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/1ef4189c77fb4cf6b3c6be0ec13f5d52 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/1ef4189c77fb4cf6b3c6be0ec13f5d52 2024-11-10T17:24:14,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/1ef4189c77fb4cf6b3c6be0ec13f5d52, entries=150, sequenceid=114, filesize=11.7 K 2024-11-10T17:24:14,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 57c77250ebf57e5a251700719e1bd916 in 1246ms, sequenceid=114, compaction requested=true 2024-11-10T17:24:14,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:14,980 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:24:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:14,980 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:24:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:14,981 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:24:14,981 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:24:14,981 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:14,981 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:14,981 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:14,981 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:14,981 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5dce24b2ca6f4fa7b71a29db2aab604b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7fba275a67c540b7a6dd76372daf5985, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7f0e5c293fd64a6c9ad90146fdca597a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/49a844ddcf0540af8af24bd46040eed0] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=51.5 K 2024-11-10T17:24:14,981 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/dd65ea156f984512a011529177db474b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/6ee48826f4ef4bd9937cd2cc6a7c6876, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c28eb65e99024920b87c136db67565b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/51ceb7c149a341e3ad68368b6ff24cd7] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=47.0 K 2024-11-10T17:24:14,981 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dce24b2ca6f4fa7b71a29db2aab604b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259450177 2024-11-10T17:24:14,981 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting dd65ea156f984512a011529177db474b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259450177 2024-11-10T17:24:14,982 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee48826f4ef4bd9937cd2cc6a7c6876, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731259451243 2024-11-10T17:24:14,982 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fba275a67c540b7a6dd76372daf5985, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731259451243 2024-11-10T17:24:14,982 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c28eb65e99024920b87c136db67565b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731259452536 2024-11-10T17:24:14,982 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0e5c293fd64a6c9ad90146fdca597a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731259452536 2024-11-10T17:24:14,982 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ceb7c149a341e3ad68368b6ff24cd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731259453395 2024-11-10T17:24:14,982 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49a844ddcf0540af8af24bd46040eed0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731259453370 2024-11-10T17:24:14,990 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:14,991 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:14,991 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/815831b44364439d921ab6c1949f6b19 is 50, key is test_row_0/A:col10/1731259453732/Put/seqid=0 2024-11-10T17:24:14,991 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/4ceaf19eafc94ab98dde689c9f8e72aa is 50, key is test_row_0/B:col10/1731259453732/Put/seqid=0 2024-11-10T17:24:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742292_1468 (size=12241) 2024-11-10T17:24:14,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742293_1469 (size=12241) 2024-11-10T17:24:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T17:24:15,272 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:15,273 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-10T17:24:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T17:24:15,274 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:15,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:15,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T17:24:15,400 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/4ceaf19eafc94ab98dde689c9f8e72aa as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4ceaf19eafc94ab98dde689c9f8e72aa 2024-11-10T17:24:15,402 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/815831b44364439d921ab6c1949f6b19 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/815831b44364439d921ab6c1949f6b19 2024-11-10T17:24:15,404 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into 4ceaf19eafc94ab98dde689c9f8e72aa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:15,404 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:15,404 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=12, startTime=1731259454980; duration=0sec 2024-11-10T17:24:15,404 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:15,404 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:15,404 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:24:15,405 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:24:15,405 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:15,406 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into 815831b44364439d921ab6c1949f6b19(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:15,406 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:15,406 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=12, startTime=1731259454980; duration=0sec 2024-11-10T17:24:15,405 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:15,406 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:15,406 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:15,406 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/81aff07caf2a4bf3ba94b2702866c8e6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/bef92cda2f83419bbac5b65251fc7792, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/114f476687374a7e924d1e1ee4702c71, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/1ef4189c77fb4cf6b3c6be0ec13f5d52] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=47.0 K 2024-11-10T17:24:15,406 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 81aff07caf2a4bf3ba94b2702866c8e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731259450177 2024-11-10T17:24:15,407 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting bef92cda2f83419bbac5b65251fc7792, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731259451243 2024-11-10T17:24:15,407 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 114f476687374a7e924d1e1ee4702c71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731259452536 2024-11-10T17:24:15,407 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef4189c77fb4cf6b3c6be0ec13f5d52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731259453395 2024-11-10T17:24:15,413 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:15,414 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/80e8bf604a504033b172598a653b0cc6 is 50, key is test_row_0/C:col10/1731259453732/Put/seqid=0 2024-11-10T17:24:15,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742294_1470 (size=12241) 2024-11-10T17:24:15,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T17:24:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:15,427 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:24:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/4c8270b54d6845cc9f51e497bf12f8f0 is 50, key is test_row_0/A:col10/1731259453742/Put/seqid=0 2024-11-10T17:24:15,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742295_1471 (size=12001) 2024-11-10T17:24:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:15,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T17:24:15,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259515658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259515782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:15,822 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/80e8bf604a504033b172598a653b0cc6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/80e8bf604a504033b172598a653b0cc6 2024-11-10T17:24:15,826 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into 80e8bf604a504033b172598a653b0cc6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:15,826 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:15,826 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=12, startTime=1731259454980; duration=0sec 2024-11-10T17:24:15,826 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:15,826 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:15,834 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/4c8270b54d6845cc9f51e497bf12f8f0 2024-11-10T17:24:15,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3c153e9c6a9249caa5c75e7a9548a1ae is 50, key is test_row_0/B:col10/1731259453742/Put/seqid=0 2024-11-10T17:24:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742296_1472 (size=12001) 2024-11-10T17:24:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T17:24:15,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259515971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:15,977 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259516001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:16,246 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3c153e9c6a9249caa5c75e7a9548a1ae 2024-11-10T17:24:16,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T17:24:16,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T17:24:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/21d31c687dd544eca545c21928cbf447 is 50, key is test_row_0/C:col10/1731259453742/Put/seqid=0 2024-11-10T17:24:16,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742297_1473 (size=12001) 2024-11-10T17:24:16,257 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/21d31c687dd544eca545c21928cbf447 2024-11-10T17:24:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/4c8270b54d6845cc9f51e497bf12f8f0 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/4c8270b54d6845cc9f51e497bf12f8f0 2024-11-10T17:24:16,264 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/4c8270b54d6845cc9f51e497bf12f8f0, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T17:24:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3c153e9c6a9249caa5c75e7a9548a1ae as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3c153e9c6a9249caa5c75e7a9548a1ae 2024-11-10T17:24:16,268 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3c153e9c6a9249caa5c75e7a9548a1ae, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T17:24:16,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/21d31c687dd544eca545c21928cbf447 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/21d31c687dd544eca545c21928cbf447 2024-11-10T17:24:16,271 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/21d31c687dd544eca545c21928cbf447, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T17:24:16,272 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 57c77250ebf57e5a251700719e1bd916 in 845ms, sequenceid=128, compaction requested=false 2024-11-10T17:24:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-10T17:24:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-10T17:24:16,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T17:24:16,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-10T17:24:16,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.0030 sec 2024-11-10T17:24:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:16,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:16,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:24:16,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/ab60e4501904435898a05b32a26c3975 is 50, key is test_row_0/A:col10/1731259456315/Put/seqid=0 2024-11-10T17:24:16,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742298_1474 (size=14541) 2024-11-10T17:24:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259516382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T17:24:16,412 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:16,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-10T17:24:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T17:24:16,414 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:16,415 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:16,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259516491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T17:24:16,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:16,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259516702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:16,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:16,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/ab60e4501904435898a05b32a26c3975 2024-11-10T17:24:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T17:24:16,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/00ca6c25e2e94ac480e2379cb30d4513 is 50, key is test_row_0/B:col10/1731259456315/Put/seqid=0 2024-11-10T17:24:16,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742299_1475 (size=12151) 2024-11-10T17:24:16,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:16,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:16,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:16,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:16,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259517021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T17:24:17,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/00ca6c25e2e94ac480e2379cb30d4513 2024-11-10T17:24:17,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/a2874192be9644c2a7e60bc751c3673c is 50, key is test_row_0/C:col10/1731259456315/Put/seqid=0 2024-11-10T17:24:17,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742300_1476 (size=12151) 2024-11-10T17:24:17,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:17,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:17,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:17,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:17,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:17,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/a2874192be9644c2a7e60bc751c3673c 2024-11-10T17:24:17,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259517542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T17:24:17,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/ab60e4501904435898a05b32a26c3975 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/ab60e4501904435898a05b32a26c3975 2024-11-10T17:24:17,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/ab60e4501904435898a05b32a26c3975, entries=200, sequenceid=154, filesize=14.2 K 2024-11-10T17:24:17,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/00ca6c25e2e94ac480e2379cb30d4513 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00ca6c25e2e94ac480e2379cb30d4513 2024-11-10T17:24:17,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00ca6c25e2e94ac480e2379cb30d4513, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T17:24:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/a2874192be9644c2a7e60bc751c3673c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/a2874192be9644c2a7e60bc751c3673c 2024-11-10T17:24:17,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/a2874192be9644c2a7e60bc751c3673c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T17:24:17,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 57c77250ebf57e5a251700719e1bd916 in 1245ms, sequenceid=154, compaction requested=true 2024-11-10T17:24:17,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:17,563 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:17,563 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:17,563 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:17,563 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:17,564 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,564 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,564 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/815831b44364439d921ab6c1949f6b19, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/4c8270b54d6845cc9f51e497bf12f8f0, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/ab60e4501904435898a05b32a26c3975] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=37.9 K 2024-11-10T17:24:17,564 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4ceaf19eafc94ab98dde689c9f8e72aa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3c153e9c6a9249caa5c75e7a9548a1ae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00ca6c25e2e94ac480e2379cb30d4513] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=35.5 K 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 815831b44364439d921ab6c1949f6b19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731259453395 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ceaf19eafc94ab98dde689c9f8e72aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731259453395 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c153e9c6a9249caa5c75e7a9548a1ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731259453742 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8270b54d6845cc9f51e497bf12f8f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731259453742 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ca6c25e2e94ac480e2379cb30d4513, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731259455642 2024-11-10T17:24:17,564 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab60e4501904435898a05b32a26c3975, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731259455642 2024-11-10T17:24:17,570 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:17,571 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:17,571 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/f61baec694604f33bd299ac59a29b0cb is 50, key is test_row_0/A:col10/1731259456315/Put/seqid=0 2024-11-10T17:24:17,571 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/bde23ecee19d476795dc63035eafe093 is 50, key is test_row_0/B:col10/1731259456315/Put/seqid=0 2024-11-10T17:24:17,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742302_1478 (size=12493) 2024-11-10T17:24:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742301_1477 (size=12493) 2024-11-10T17:24:17,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T17:24:17,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,638 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:24:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d5e3efde9a2c440889059604bcfa0804 is 50, key is test_row_0/A:col10/1731259456381/Put/seqid=0 2024-11-10T17:24:17,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742303_1479 (size=12151) 2024-11-10T17:24:17,981 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/f61baec694604f33bd299ac59a29b0cb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/f61baec694604f33bd299ac59a29b0cb 2024-11-10T17:24:17,985 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into f61baec694604f33bd299ac59a29b0cb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:17,985 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:17,985 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=13, startTime=1731259457562; duration=0sec 2024-11-10T17:24:17,985 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:17,985 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:17,985 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:17,986 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:17,986 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:17,986 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:17,986 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/80e8bf604a504033b172598a653b0cc6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/21d31c687dd544eca545c21928cbf447, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/a2874192be9644c2a7e60bc751c3673c] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=35.5 K 2024-11-10T17:24:17,986 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80e8bf604a504033b172598a653b0cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731259453395 2024-11-10T17:24:17,986 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d31c687dd544eca545c21928cbf447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731259453742 2024-11-10T17:24:17,987 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2874192be9644c2a7e60bc751c3673c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731259455642 2024-11-10T17:24:17,993 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:17,993 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/6befb8e72a43425d8c555e570dcb5598 is 50, key is test_row_0/C:col10/1731259456315/Put/seqid=0 2024-11-10T17:24:17,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742304_1480 (size=12493) 2024-11-10T17:24:18,002 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/bde23ecee19d476795dc63035eafe093 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/bde23ecee19d476795dc63035eafe093 2024-11-10T17:24:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:18,005 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into bde23ecee19d476795dc63035eafe093(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:18,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:18,005 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:18,006 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=13, startTime=1731259457563; duration=0sec 2024-11-10T17:24:18,006 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:18,006 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:18,046 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d5e3efde9a2c440889059604bcfa0804 2024-11-10T17:24:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/5c9e21356c514fcf8f48ad1fee3ab088 is 50, key is test_row_0/B:col10/1731259456381/Put/seqid=0 2024-11-10T17:24:18,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742305_1481 (size=12151) 2024-11-10T17:24:18,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259518178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259518302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,401 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/6befb8e72a43425d8c555e570dcb5598 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6befb8e72a43425d8c555e570dcb5598 2024-11-10T17:24:18,407 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into 6befb8e72a43425d8c555e570dcb5598(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:18,407 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:18,407 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=13, startTime=1731259457563; duration=0sec 2024-11-10T17:24:18,407 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:18,407 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:18,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259518442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259518442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,449 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,449 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259518451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,461 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/5c9e21356c514fcf8f48ad1fee3ab088 2024-11-10T17:24:18,460 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c578df881084497592ef54bf4e041fb3 is 50, key is test_row_0/C:col10/1731259456381/Put/seqid=0 2024-11-10T17:24:18,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742306_1482 (size=12151) 2024-11-10T17:24:18,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259518512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259518551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,560 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T17:24:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259518831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:18,876 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c578df881084497592ef54bf4e041fb3 2024-11-10T17:24:18,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d5e3efde9a2c440889059604bcfa0804 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d5e3efde9a2c440889059604bcfa0804 2024-11-10T17:24:18,885 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d5e3efde9a2c440889059604bcfa0804, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T17:24:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/5c9e21356c514fcf8f48ad1fee3ab088 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/5c9e21356c514fcf8f48ad1fee3ab088 2024-11-10T17:24:18,889 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/5c9e21356c514fcf8f48ad1fee3ab088, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T17:24:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c578df881084497592ef54bf4e041fb3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c578df881084497592ef54bf4e041fb3 2024-11-10T17:24:18,893 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c578df881084497592ef54bf4e041fb3, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T17:24:18,894 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 57c77250ebf57e5a251700719e1bd916 in 1256ms, sequenceid=165, compaction requested=false 2024-11-10T17:24:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-10T17:24:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-10T17:24:18,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T17:24:18,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4810 sec 2024-11-10T17:24:18,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.4870 sec 2024-11-10T17:24:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:19,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:24:19,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/94e8a893d58348538bbba17ba28dd505 is 50, key is test_row_0/A:col10/1731259459355/Put/seqid=0 2024-11-10T17:24:19,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742307_1483 (size=14541) 2024-11-10T17:24:19,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/94e8a893d58348538bbba17ba28dd505 2024-11-10T17:24:19,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/434a794f99bd4fdcaef22bfeb0eb41de is 50, key is test_row_0/B:col10/1731259459355/Put/seqid=0 2024-11-10T17:24:19,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742308_1484 (size=12151) 2024-11-10T17:24:19,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/434a794f99bd4fdcaef22bfeb0eb41de 2024-11-10T17:24:19,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/3b7e550c6077438881cb8e1b8b85c177 is 50, key is test_row_0/C:col10/1731259459355/Put/seqid=0 2024-11-10T17:24:19,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742309_1485 (size=12151) 2024-11-10T17:24:19,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259519427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259519542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:19,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259519751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:19,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/3b7e550c6077438881cb8e1b8b85c177 2024-11-10T17:24:19,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/94e8a893d58348538bbba17ba28dd505 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/94e8a893d58348538bbba17ba28dd505 2024-11-10T17:24:19,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/94e8a893d58348538bbba17ba28dd505, entries=200, sequenceid=194, filesize=14.2 K 2024-11-10T17:24:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/434a794f99bd4fdcaef22bfeb0eb41de as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/434a794f99bd4fdcaef22bfeb0eb41de 2024-11-10T17:24:19,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/434a794f99bd4fdcaef22bfeb0eb41de, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T17:24:19,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/3b7e550c6077438881cb8e1b8b85c177 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3b7e550c6077438881cb8e1b8b85c177 2024-11-10T17:24:19,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3b7e550c6077438881cb8e1b8b85c177, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T17:24:19,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 57c77250ebf57e5a251700719e1bd916 in 490ms, sequenceid=194, compaction requested=true 2024-11-10T17:24:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:19,847 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T17:24:19,847 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:19,848 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:19,849 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:19,849 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:19,849 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/bde23ecee19d476795dc63035eafe093, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/5c9e21356c514fcf8f48ad1fee3ab088, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/434a794f99bd4fdcaef22bfeb0eb41de] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=35.9 K 2024-11-10T17:24:19,849 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:19,849 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:19,849 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:19,849 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/f61baec694604f33bd299ac59a29b0cb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d5e3efde9a2c440889059604bcfa0804, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/94e8a893d58348538bbba17ba28dd505] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=38.3 K 2024-11-10T17:24:19,850 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting f61baec694604f33bd299ac59a29b0cb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731259455642 2024-11-10T17:24:19,850 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting bde23ecee19d476795dc63035eafe093, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731259455642 2024-11-10T17:24:19,850 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5e3efde9a2c440889059604bcfa0804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731259456367 2024-11-10T17:24:19,850 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c9e21356c514fcf8f48ad1fee3ab088, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731259456367 2024-11-10T17:24:19,850 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94e8a893d58348538bbba17ba28dd505, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259458151 2024-11-10T17:24:19,851 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 434a794f99bd4fdcaef22bfeb0eb41de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259458164 2024-11-10T17:24:19,875 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:19,876 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/17e357716a4843849bad930ee287cbae is 50, key is test_row_0/B:col10/1731259459355/Put/seqid=0 2024-11-10T17:24:19,899 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:19,899 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/30356236ede0498a83a4e10633c402b5 is 50, key is test_row_0/A:col10/1731259459355/Put/seqid=0 2024-11-10T17:24:19,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742310_1486 (size=12595) 2024-11-10T17:24:19,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742311_1487 (size=12595) 2024-11-10T17:24:19,961 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/17e357716a4843849bad930ee287cbae as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/17e357716a4843849bad930ee287cbae 2024-11-10T17:24:19,967 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into 17e357716a4843849bad930ee287cbae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:19,967 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:19,967 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=13, startTime=1731259459847; duration=0sec 2024-11-10T17:24:19,967 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:19,967 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:19,967 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:19,968 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:19,968 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:19,968 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:19,968 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6befb8e72a43425d8c555e570dcb5598, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c578df881084497592ef54bf4e041fb3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3b7e550c6077438881cb8e1b8b85c177] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=35.9 K 2024-11-10T17:24:19,968 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6befb8e72a43425d8c555e570dcb5598, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731259455642 2024-11-10T17:24:19,969 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c578df881084497592ef54bf4e041fb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731259456367 2024-11-10T17:24:19,970 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7e550c6077438881cb8e1b8b85c177, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259458164 2024-11-10T17:24:19,990 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:19,991 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/5ef81b48c2b4428a8f021ae4235558de is 50, key is test_row_0/C:col10/1731259459355/Put/seqid=0 2024-11-10T17:24:20,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742312_1488 (size=12595) 2024-11-10T17:24:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:20,065 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:20,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:24:20,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d816149303364f2f92cf7c126f7f03e4 is 50, key is test_row_0/A:col10/1731259459405/Put/seqid=0 2024-11-10T17:24:20,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742313_1489 (size=14541) 2024-11-10T17:24:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259520325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:20,366 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/30356236ede0498a83a4e10633c402b5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/30356236ede0498a83a4e10633c402b5 2024-11-10T17:24:20,371 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into 30356236ede0498a83a4e10633c402b5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:20,371 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:20,371 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=13, startTime=1731259459847; duration=0sec 2024-11-10T17:24:20,371 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:20,371 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:20,410 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/5ef81b48c2b4428a8f021ae4235558de as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5ef81b48c2b4428a8f021ae4235558de 2024-11-10T17:24:20,414 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into 5ef81b48c2b4428a8f021ae4235558de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:20,414 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:20,414 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=13, startTime=1731259459847; duration=0sec 2024-11-10T17:24:20,415 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:20,415 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:20,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259520441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:20,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d816149303364f2f92cf7c126f7f03e4 2024-11-10T17:24:20,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/1027c23e822f452f919dcf5bc7252b15 is 50, key is test_row_0/B:col10/1731259459405/Put/seqid=0 2024-11-10T17:24:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742314_1490 (size=12151) 2024-11-10T17:24:20,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259520571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:20,578 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T17:24:20,582 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:20,583 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-10T17:24:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T17:24:20,585 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:20,585 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:20,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:20,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259520652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T17:24:20,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T17:24:20,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:20,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:20,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:20,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:20,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:20,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T17:24:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:20,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T17:24:20,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/1027c23e822f452f919dcf5bc7252b15 2024-11-10T17:24:20,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/5418c8e6a4e449b18821dd6005b20aaa is 50, key is test_row_0/C:col10/1731259459405/Put/seqid=0 2024-11-10T17:24:20,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742315_1491 (size=12151) 2024-11-10T17:24:20,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/5418c8e6a4e449b18821dd6005b20aaa 2024-11-10T17:24:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d816149303364f2f92cf7c126f7f03e4 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d816149303364f2f92cf7c126f7f03e4 2024-11-10T17:24:20,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d816149303364f2f92cf7c126f7f03e4, entries=200, sequenceid=206, filesize=14.2 K 2024-11-10T17:24:20,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/1027c23e822f452f919dcf5bc7252b15 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/1027c23e822f452f919dcf5bc7252b15 2024-11-10T17:24:20,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/1027c23e822f452f919dcf5bc7252b15, entries=150, sequenceid=206, filesize=11.9 K 2024-11-10T17:24:20,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/5418c8e6a4e449b18821dd6005b20aaa as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5418c8e6a4e449b18821dd6005b20aaa 2024-11-10T17:24:20,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5418c8e6a4e449b18821dd6005b20aaa, entries=150, sequenceid=206, filesize=11.9 K 2024-11-10T17:24:20,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 57c77250ebf57e5a251700719e1bd916 in 884ms, sequenceid=206, compaction requested=false 2024-11-10T17:24:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:20,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:20,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:24:20,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5063feb09d5a485a9f95f5df2b85b5e7 is 50, key is test_row_0/A:col10/1731259460278/Put/seqid=0 2024-11-10T17:24:20,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742316_1492 (size=14541) 2024-11-10T17:24:20,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5063feb09d5a485a9f95f5df2b85b5e7 2024-11-10T17:24:21,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/288d900ed00c43de955137f226684c99 is 50, key is test_row_0/B:col10/1731259460278/Put/seqid=0 2024-11-10T17:24:21,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742317_1493 (size=12151) 2024-11-10T17:24:21,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T17:24:21,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:21,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259521038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259521161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:21,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T17:24:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T17:24:21,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T17:24:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:21,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259521372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:21,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/288d900ed00c43de955137f226684c99 2024-11-10T17:24:21,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/12583befee454049bfbc81071b8dc39f is 50, key is test_row_0/C:col10/1731259460278/Put/seqid=0 2024-11-10T17:24:21,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742318_1494 (size=12151) 2024-11-10T17:24:21,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/12583befee454049bfbc81071b8dc39f 2024-11-10T17:24:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5063feb09d5a485a9f95f5df2b85b5e7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5063feb09d5a485a9f95f5df2b85b5e7 2024-11-10T17:24:21,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5063feb09d5a485a9f95f5df2b85b5e7, entries=200, sequenceid=234, filesize=14.2 K 2024-11-10T17:24:21,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/288d900ed00c43de955137f226684c99 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/288d900ed00c43de955137f226684c99 2024-11-10T17:24:21,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/288d900ed00c43de955137f226684c99, entries=150, sequenceid=234, filesize=11.9 K 2024-11-10T17:24:21,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/12583befee454049bfbc81071b8dc39f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/12583befee454049bfbc81071b8dc39f 2024-11-10T17:24:21,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/12583befee454049bfbc81071b8dc39f, entries=150, sequenceid=234, filesize=11.9 K 2024-11-10T17:24:21,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 57c77250ebf57e5a251700719e1bd916 in 507ms, sequenceid=234, compaction requested=true 2024-11-10T17:24:21,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:21,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:21,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:21,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:21,484 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:21,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:21,484 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:21,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:21,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:21,487 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:21,487 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:21,487 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:21,487 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:21,487 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,487 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,487 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/30356236ede0498a83a4e10633c402b5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d816149303364f2f92cf7c126f7f03e4, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5063feb09d5a485a9f95f5df2b85b5e7] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=40.7 K 2024-11-10T17:24:21,488 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/17e357716a4843849bad930ee287cbae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/1027c23e822f452f919dcf5bc7252b15, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/288d900ed00c43de955137f226684c99] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=36.0 K 2024-11-10T17:24:21,488 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 17e357716a4843849bad930ee287cbae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259458164 2024-11-10T17:24:21,488 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30356236ede0498a83a4e10633c402b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259458164 2024-11-10T17:24:21,488 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 1027c23e822f452f919dcf5bc7252b15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259459405 2024-11-10T17:24:21,488 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting d816149303364f2f92cf7c126f7f03e4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259459394 2024-11-10T17:24:21,489 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 288d900ed00c43de955137f226684c99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731259460278 2024-11-10T17:24:21,489 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5063feb09d5a485a9f95f5df2b85b5e7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731259460278 2024-11-10T17:24:21,498 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:21,499 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/b99b92f4134f4c609d8bba68c7470a29 is 50, key is test_row_0/A:col10/1731259460278/Put/seqid=0 2024-11-10T17:24:21,502 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:21,503 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3516ac0fa009400c901e48a66b201370 is 50, key is test_row_0/B:col10/1731259460278/Put/seqid=0 2024-11-10T17:24:21,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T17:24:21,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,505 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T17:24:21,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742319_1495 (size=12697) 2024-11-10T17:24:21,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/e2ff4f46a7c84276b0c64a2daaa29aef is 50, key is test_row_1/A:col10/1731259461001/Put/seqid=0 2024-11-10T17:24:21,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742320_1496 (size=12697) 2024-11-10T17:24:21,530 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3516ac0fa009400c901e48a66b201370 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3516ac0fa009400c901e48a66b201370 2024-11-10T17:24:21,540 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into 3516ac0fa009400c901e48a66b201370(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:21,540 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:21,540 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=13, startTime=1731259461484; duration=0sec 2024-11-10T17:24:21,540 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:21,540 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:21,541 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:21,541 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:21,542 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:21,542 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:21,542 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5ef81b48c2b4428a8f021ae4235558de, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5418c8e6a4e449b18821dd6005b20aaa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/12583befee454049bfbc81071b8dc39f] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=36.0 K 2024-11-10T17:24:21,542 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef81b48c2b4428a8f021ae4235558de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731259458164 2024-11-10T17:24:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742321_1497 (size=9757) 2024-11-10T17:24:21,542 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 5418c8e6a4e449b18821dd6005b20aaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731259459405 2024-11-10T17:24:21,543 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 12583befee454049bfbc81071b8dc39f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731259460278 2024-11-10T17:24:21,543 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/e2ff4f46a7c84276b0c64a2daaa29aef 2024-11-10T17:24:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/4c591835680e40c78818d3c5754daa97 is 50, key is test_row_1/B:col10/1731259461001/Put/seqid=0 2024-11-10T17:24:21,572 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:21,573 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/60652cae52a3408bb6f832fbb05bef3d is 50, key is test_row_0/C:col10/1731259460278/Put/seqid=0 2024-11-10T17:24:21,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742322_1498 (size=9757) 2024-11-10T17:24:21,616 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/4c591835680e40c78818d3c5754daa97 2024-11-10T17:24:21,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/63550f6ba45b4eac9299cb94d5b9658a is 50, key is test_row_1/C:col10/1731259461001/Put/seqid=0 2024-11-10T17:24:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742323_1499 (size=12697) 2024-11-10T17:24:21,659 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/60652cae52a3408bb6f832fbb05bef3d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/60652cae52a3408bb6f832fbb05bef3d 2024-11-10T17:24:21,665 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into 60652cae52a3408bb6f832fbb05bef3d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:21,665 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:21,665 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=13, startTime=1731259461484; duration=0sec 2024-11-10T17:24:21,665 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:21,665 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:21,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742324_1500 (size=9757) 2024-11-10T17:24:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:21,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T17:24:21,915 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/b99b92f4134f4c609d8bba68c7470a29 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/b99b92f4134f4c609d8bba68c7470a29 2024-11-10T17:24:21,919 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into b99b92f4134f4c609d8bba68c7470a29(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:21,920 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:21,920 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=13, startTime=1731259461484; duration=0sec 2024-11-10T17:24:21,920 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:21,920 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:21,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259521920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:22,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259522041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:22,095 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/63550f6ba45b4eac9299cb94d5b9658a 2024-11-10T17:24:22,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/e2ff4f46a7c84276b0c64a2daaa29aef as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/e2ff4f46a7c84276b0c64a2daaa29aef 2024-11-10T17:24:22,103 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/e2ff4f46a7c84276b0c64a2daaa29aef, entries=100, sequenceid=244, filesize=9.5 K 2024-11-10T17:24:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/4c591835680e40c78818d3c5754daa97 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4c591835680e40c78818d3c5754daa97 2024-11-10T17:24:22,109 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4c591835680e40c78818d3c5754daa97, entries=100, sequenceid=244, filesize=9.5 K 2024-11-10T17:24:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/63550f6ba45b4eac9299cb94d5b9658a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/63550f6ba45b4eac9299cb94d5b9658a 2024-11-10T17:24:22,113 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/63550f6ba45b4eac9299cb94d5b9658a, entries=100, sequenceid=244, filesize=9.5 K 2024-11-10T17:24:22,114 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 57c77250ebf57e5a251700719e1bd916 in 610ms, sequenceid=244, compaction requested=false 2024-11-10T17:24:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-10T17:24:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-10T17:24:22,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T17:24:22,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5300 sec 2024-11-10T17:24:22,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5370 sec 2024-11-10T17:24:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:22,267 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:22,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T17:24:22,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/623f13c4b6c34ef383fe841d9a807066 is 50, key is test_row_0/A:col10/1731259461894/Put/seqid=0 2024-11-10T17:24:22,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742325_1501 (size=14741) 2024-11-10T17:24:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259522302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259522411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259522622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:22,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/623f13c4b6c34ef383fe841d9a807066 2024-11-10T17:24:22,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/9d25a52afebc44719fbb8a1ae89a4457 is 50, key is test_row_0/B:col10/1731259461894/Put/seqid=0 2024-11-10T17:24:22,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742326_1502 (size=12301) 2024-11-10T17:24:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T17:24:22,732 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:22,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-10T17:24:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T17:24:22,734 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:22,735 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:22,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T17:24:22,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T17:24:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:22,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:22,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259522942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:23,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T17:24:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T17:24:23,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/9d25a52afebc44719fbb8a1ae89a4457 2024-11-10T17:24:23,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/8a46c399d63449a2ae1b2925e7eb1fcf is 50, key is test_row_0/C:col10/1731259461894/Put/seqid=0 2024-11-10T17:24:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742327_1503 (size=12301) 2024-11-10T17:24:23,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T17:24:23,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:23,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T17:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T17:24:23,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259523461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:23,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T17:24:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:23,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:23,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/8a46c399d63449a2ae1b2925e7eb1fcf 2024-11-10T17:24:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/623f13c4b6c34ef383fe841d9a807066 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/623f13c4b6c34ef383fe841d9a807066 2024-11-10T17:24:23,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/623f13c4b6c34ef383fe841d9a807066, entries=200, sequenceid=274, filesize=14.4 K 2024-11-10T17:24:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/9d25a52afebc44719fbb8a1ae89a4457 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/9d25a52afebc44719fbb8a1ae89a4457 2024-11-10T17:24:23,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/9d25a52afebc44719fbb8a1ae89a4457, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T17:24:23,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/8a46c399d63449a2ae1b2925e7eb1fcf as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8a46c399d63449a2ae1b2925e7eb1fcf 2024-11-10T17:24:23,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8a46c399d63449a2ae1b2925e7eb1fcf, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T17:24:23,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 57c77250ebf57e5a251700719e1bd916 in 1270ms, sequenceid=274, compaction requested=true 2024-11-10T17:24:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:23,537 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:23,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:23,537 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:23,538 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:23,538 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:23,538 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:23,538 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:23,538 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,538 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,538 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3516ac0fa009400c901e48a66b201370, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4c591835680e40c78818d3c5754daa97, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/9d25a52afebc44719fbb8a1ae89a4457] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=33.9 K 2024-11-10T17:24:23,538 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/b99b92f4134f4c609d8bba68c7470a29, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/e2ff4f46a7c84276b0c64a2daaa29aef, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/623f13c4b6c34ef383fe841d9a807066] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=36.3 K 2024-11-10T17:24:23,538 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3516ac0fa009400c901e48a66b201370, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731259460278 2024-11-10T17:24:23,538 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99b92f4134f4c609d8bba68c7470a29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731259460278 2024-11-10T17:24:23,539 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c591835680e40c78818d3c5754daa97, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731259461001 2024-11-10T17:24:23,539 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ff4f46a7c84276b0c64a2daaa29aef, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731259461001 2024-11-10T17:24:23,541 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 623f13c4b6c34ef383fe841d9a807066, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731259461854 2024-11-10T17:24:23,542 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d25a52afebc44719fbb8a1ae89a4457, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731259461854 2024-11-10T17:24:23,548 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:23,549 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/00819ce45aad4915a7d8ea083fded23c is 50, key is test_row_0/B:col10/1731259461894/Put/seqid=0 2024-11-10T17:24:23,550 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:23,551 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5f8fe254a5f34ed1897882f772336e96 is 50, key is test_row_0/A:col10/1731259461894/Put/seqid=0 2024-11-10T17:24:23,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742328_1504 (size=12949) 2024-11-10T17:24:23,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742329_1505 (size=12949) 2024-11-10T17:24:23,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T17:24:23,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,666 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T17:24:23,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d4452f9668364a10b82c12fb8af8a08f is 50, key is test_row_0/A:col10/1731259462284/Put/seqid=0 2024-11-10T17:24:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742330_1506 (size=9857) 2024-11-10T17:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T17:24:23,959 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/00819ce45aad4915a7d8ea083fded23c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00819ce45aad4915a7d8ea083fded23c 2024-11-10T17:24:23,966 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into 00819ce45aad4915a7d8ea083fded23c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:23,966 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:23,966 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=13, startTime=1731259463537; duration=0sec 2024-11-10T17:24:23,966 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:23,966 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:23,966 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:23,968 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:23,968 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:23,968 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:23,968 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/60652cae52a3408bb6f832fbb05bef3d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/63550f6ba45b4eac9299cb94d5b9658a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8a46c399d63449a2ae1b2925e7eb1fcf] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=33.9 K 2024-11-10T17:24:23,968 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 60652cae52a3408bb6f832fbb05bef3d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731259460278 2024-11-10T17:24:23,968 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5f8fe254a5f34ed1897882f772336e96 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5f8fe254a5f34ed1897882f772336e96 2024-11-10T17:24:23,969 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 63550f6ba45b4eac9299cb94d5b9658a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731259461001 2024-11-10T17:24:23,969 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a46c399d63449a2ae1b2925e7eb1fcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731259461854 2024-11-10T17:24:23,974 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into 5f8fe254a5f34ed1897882f772336e96(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:23,974 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:23,974 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=13, startTime=1731259463537; duration=0sec 2024-11-10T17:24:23,974 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:23,974 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:23,979 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:23,980 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/fc68de03296f4234bb86f850824a8d11 is 50, key is test_row_0/C:col10/1731259461894/Put/seqid=0 2024-11-10T17:24:23,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742331_1507 (size=12949) 2024-11-10T17:24:23,991 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/fc68de03296f4234bb86f850824a8d11 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc68de03296f4234bb86f850824a8d11 2024-11-10T17:24:23,996 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into fc68de03296f4234bb86f850824a8d11(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:23,996 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:23,996 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=13, startTime=1731259463537; duration=0sec 2024-11-10T17:24:23,996 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:23,996 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:24,079 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d4452f9668364a10b82c12fb8af8a08f 2024-11-10T17:24:24,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3f15447ee40e4500995f6e3fc4a1378f is 50, key is test_row_0/B:col10/1731259462284/Put/seqid=0 2024-11-10T17:24:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742332_1508 (size=9857) 2024-11-10T17:24:24,103 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3f15447ee40e4500995f6e3fc4a1378f 2024-11-10T17:24:24,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/9b81e63970254d02aaa19b18de3c906d is 50, key is test_row_0/C:col10/1731259462284/Put/seqid=0 2024-11-10T17:24:24,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742333_1509 (size=9857) 2024-11-10T17:24:24,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/9b81e63970254d02aaa19b18de3c906d 2024-11-10T17:24:24,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/d4452f9668364a10b82c12fb8af8a08f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d4452f9668364a10b82c12fb8af8a08f 2024-11-10T17:24:24,137 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d4452f9668364a10b82c12fb8af8a08f, entries=100, sequenceid=283, filesize=9.6 K 2024-11-10T17:24:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/3f15447ee40e4500995f6e3fc4a1378f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3f15447ee40e4500995f6e3fc4a1378f 2024-11-10T17:24:24,144 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3f15447ee40e4500995f6e3fc4a1378f, entries=100, sequenceid=283, filesize=9.6 K 2024-11-10T17:24:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/9b81e63970254d02aaa19b18de3c906d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/9b81e63970254d02aaa19b18de3c906d 2024-11-10T17:24:24,150 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/9b81e63970254d02aaa19b18de3c906d, entries=100, sequenceid=283, filesize=9.6 K 2024-11-10T17:24:24,152 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 57c77250ebf57e5a251700719e1bd916 in 485ms, sequenceid=283, compaction requested=false 2024-11-10T17:24:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-10T17:24:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-10T17:24:24,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T17:24:24,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-10T17:24:24,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4240 sec 2024-11-10T17:24:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:24,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:24,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:24:24,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/3c0c1aedd8234d4b992f3dbc22189eec is 50, key is test_row_0/A:col10/1731259464524/Put/seqid=0 2024-11-10T17:24:24,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742334_1510 (size=12301) 2024-11-10T17:24:24,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259524712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259524720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259524832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:24,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259524843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T17:24:24,882 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:24,882 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T17:24:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T17:24:24,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:24,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:24,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:24,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/3c0c1aedd8234d4b992f3dbc22189eec 2024-11-10T17:24:24,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/c8c3dba820844d07bc9bb3485c3bfcb7 is 50, key is test_row_0/B:col10/1731259464524/Put/seqid=0 2024-11-10T17:24:24,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742335_1511 (size=12301) 2024-11-10T17:24:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T17:24:25,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T17:24:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259525052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259525062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:25,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T17:24:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T17:24:25,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T17:24:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:25,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/c8c3dba820844d07bc9bb3485c3bfcb7 2024-11-10T17:24:25,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259525371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:25,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/8eae05bbcfc1463580c594fd868efc7e is 50, key is test_row_0/C:col10/1731259464524/Put/seqid=0 2024-11-10T17:24:25,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742336_1512 (size=12301) 2024-11-10T17:24:25,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/8eae05bbcfc1463580c594fd868efc7e 2024-11-10T17:24:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259525384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/3c0c1aedd8234d4b992f3dbc22189eec as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/3c0c1aedd8234d4b992f3dbc22189eec 2024-11-10T17:24:25,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/3c0c1aedd8234d4b992f3dbc22189eec, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T17:24:25,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/c8c3dba820844d07bc9bb3485c3bfcb7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c8c3dba820844d07bc9bb3485c3bfcb7 2024-11-10T17:24:25,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c8c3dba820844d07bc9bb3485c3bfcb7, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T17:24:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/8eae05bbcfc1463580c594fd868efc7e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8eae05bbcfc1463580c594fd868efc7e 2024-11-10T17:24:25,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8eae05bbcfc1463580c594fd868efc7e, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T17:24:25,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 57c77250ebf57e5a251700719e1bd916 in 877ms, sequenceid=297, compaction requested=true 2024-11-10T17:24:25,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:25,404 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:25,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:25,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:25,404 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:25,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:25,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:25,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:25,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:25,406 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:25,407 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:25,407 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,407 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5f8fe254a5f34ed1897882f772336e96, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d4452f9668364a10b82c12fb8af8a08f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/3c0c1aedd8234d4b992f3dbc22189eec] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=34.3 K 2024-11-10T17:24:25,407 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:25,407 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:25,407 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,407 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00819ce45aad4915a7d8ea083fded23c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3f15447ee40e4500995f6e3fc4a1378f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c8c3dba820844d07bc9bb3485c3bfcb7] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=34.3 K 2024-11-10T17:24:25,408 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 00819ce45aad4915a7d8ea083fded23c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731259461854 2024-11-10T17:24:25,408 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f8fe254a5f34ed1897882f772336e96, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731259461854 2024-11-10T17:24:25,408 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f15447ee40e4500995f6e3fc4a1378f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731259462284 2024-11-10T17:24:25,408 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4452f9668364a10b82c12fb8af8a08f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731259462284 2024-11-10T17:24:25,408 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c3dba820844d07bc9bb3485c3bfcb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731259464507 2024-11-10T17:24:25,409 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c0c1aedd8234d4b992f3dbc22189eec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731259464507 2024-11-10T17:24:25,416 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:25,417 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/c94c288e66cf4ceebcb467e8d50c7e53 is 50, key is test_row_0/A:col10/1731259464524/Put/seqid=0 2024-11-10T17:24:25,422 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:25,423 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/7b2a0232a91e46aab985e5455500b43b is 50, key is test_row_0/B:col10/1731259464524/Put/seqid=0 2024-11-10T17:24:25,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742337_1513 (size=13051) 2024-11-10T17:24:25,468 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/c94c288e66cf4ceebcb467e8d50c7e53 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/c94c288e66cf4ceebcb467e8d50c7e53 2024-11-10T17:24:25,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742338_1514 (size=13051) 2024-11-10T17:24:25,473 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into c94c288e66cf4ceebcb467e8d50c7e53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:25,473 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:25,473 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=13, startTime=1731259465404; duration=0sec 2024-11-10T17:24:25,473 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:25,473 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:25,473 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:25,474 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/7b2a0232a91e46aab985e5455500b43b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7b2a0232a91e46aab985e5455500b43b 2024-11-10T17:24:25,475 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:25,475 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:25,475 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,475 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc68de03296f4234bb86f850824a8d11, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/9b81e63970254d02aaa19b18de3c906d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8eae05bbcfc1463580c594fd868efc7e] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=34.3 K 2024-11-10T17:24:25,475 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc68de03296f4234bb86f850824a8d11, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731259461854 2024-11-10T17:24:25,477 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b81e63970254d02aaa19b18de3c906d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731259462284 2024-11-10T17:24:25,478 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eae05bbcfc1463580c594fd868efc7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731259464507 2024-11-10T17:24:25,481 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into 7b2a0232a91e46aab985e5455500b43b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:25,481 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:25,481 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=13, startTime=1731259465404; duration=0sec 2024-11-10T17:24:25,481 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:25,481 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:25,487 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:25,488 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/f82447006c1243f7a7044bca9d50ad76 is 50, key is test_row_0/C:col10/1731259464524/Put/seqid=0 2024-11-10T17:24:25,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T17:24:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:25,501 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:24:25,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742339_1515 (size=13051) 2024-11-10T17:24:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5be05660a9424145b5cafcf6a965ea06 is 50, key is test_row_0/A:col10/1731259464718/Put/seqid=0 2024-11-10T17:24:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T17:24:25,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742340_1516 (size=12301) 2024-11-10T17:24:25,521 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5be05660a9424145b5cafcf6a965ea06 2024-11-10T17:24:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/98ab2194196b46f19472a86ea4544943 is 50, key is test_row_0/B:col10/1731259464718/Put/seqid=0 2024-11-10T17:24:25,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742341_1517 (size=12301) 2024-11-10T17:24:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:25,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:25,910 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/f82447006c1243f7a7044bca9d50ad76 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/f82447006c1243f7a7044bca9d50ad76 2024-11-10T17:24:25,923 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into f82447006c1243f7a7044bca9d50ad76(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:25,923 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:25,923 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=13, startTime=1731259465405; duration=0sec 2024-11-10T17:24:25,923 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:25,923 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:25,942 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/98ab2194196b46f19472a86ea4544943 2024-11-10T17:24:25,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259525935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:25,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259525946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:25,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:25,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/b576d2aafed64fa4b71f379c16477326 is 50, key is test_row_0/C:col10/1731259464718/Put/seqid=0 2024-11-10T17:24:26,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742342_1518 (size=12301) 2024-11-10T17:24:26,018 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/b576d2aafed64fa4b71f379c16477326 2024-11-10T17:24:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T17:24:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/5be05660a9424145b5cafcf6a965ea06 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5be05660a9424145b5cafcf6a965ea06 2024-11-10T17:24:26,027 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5be05660a9424145b5cafcf6a965ea06, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T17:24:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/98ab2194196b46f19472a86ea4544943 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/98ab2194196b46f19472a86ea4544943 2024-11-10T17:24:26,038 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/98ab2194196b46f19472a86ea4544943, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T17:24:26,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/b576d2aafed64fa4b71f379c16477326 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/b576d2aafed64fa4b71f379c16477326 2024-11-10T17:24:26,052 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/b576d2aafed64fa4b71f379c16477326, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T17:24:26,053 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 57c77250ebf57e5a251700719e1bd916 in 552ms, sequenceid=324, compaction requested=false 2024-11-10T17:24:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:26,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:26,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T17:24:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T17:24:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:26,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:24:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:26,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T17:24:26,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1700 sec 2024-11-10T17:24:26,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1760 sec 2024-11-10T17:24:26,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/dd1159206f1248389897de048517e209 is 50, key is test_row_0/A:col10/1731259465918/Put/seqid=0 2024-11-10T17:24:26,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742343_1519 (size=14741) 2024-11-10T17:24:26,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259526234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259526234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259526352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259526352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/dd1159206f1248389897de048517e209 2024-11-10T17:24:26,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/b85f7637425a40b2a79f90fd9ef3a692 is 50, key is test_row_0/B:col10/1731259465918/Put/seqid=0 2024-11-10T17:24:26,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742344_1520 (size=12301) 2024-11-10T17:24:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259526572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259526572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259526892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259526892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:26,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/b85f7637425a40b2a79f90fd9ef3a692 2024-11-10T17:24:26,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c3cba384090843ca870556fabf186fcc is 50, key is test_row_0/C:col10/1731259465918/Put/seqid=0 2024-11-10T17:24:26,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742345_1521 (size=12301) 2024-11-10T17:24:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T17:24:27,044 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:27,045 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-10T17:24:27,049 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:27,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:27,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T17:24:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T17:24:27,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c3cba384090843ca870556fabf186fcc 2024-11-10T17:24:27,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/dd1159206f1248389897de048517e209 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/dd1159206f1248389897de048517e209 2024-11-10T17:24:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T17:24:27,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/dd1159206f1248389897de048517e209, entries=200, sequenceid=337, filesize=14.4 K 2024-11-10T17:24:27,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/b85f7637425a40b2a79f90fd9ef3a692 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/b85f7637425a40b2a79f90fd9ef3a692 2024-11-10T17:24:27,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/b85f7637425a40b2a79f90fd9ef3a692, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T17:24:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c3cba384090843ca870556fabf186fcc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c3cba384090843ca870556fabf186fcc 2024-11-10T17:24:27,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c3cba384090843ca870556fabf186fcc, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T17:24:27,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 57c77250ebf57e5a251700719e1bd916 in 1332ms, sequenceid=337, compaction requested=true 2024-11-10T17:24:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:27,387 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:27,387 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c77250ebf57e5a251700719e1bd916:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:27,388 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:27,388 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/B is initiating minor compaction (all files) 2024-11-10T17:24:27,388 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/B in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,388 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7b2a0232a91e46aab985e5455500b43b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/98ab2194196b46f19472a86ea4544943, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/b85f7637425a40b2a79f90fd9ef3a692] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=36.8 K 2024-11-10T17:24:27,388 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:27,388 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/A is initiating minor compaction (all files) 2024-11-10T17:24:27,388 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/A in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,389 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/c94c288e66cf4ceebcb467e8d50c7e53, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5be05660a9424145b5cafcf6a965ea06, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/dd1159206f1248389897de048517e209] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=39.2 K 2024-11-10T17:24:27,389 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94c288e66cf4ceebcb467e8d50c7e53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731259464507 2024-11-10T17:24:27,389 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2a0232a91e46aab985e5455500b43b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731259464507 2024-11-10T17:24:27,389 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be05660a9424145b5cafcf6a965ea06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731259464703 2024-11-10T17:24:27,390 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ab2194196b46f19472a86ea4544943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731259464703 2024-11-10T17:24:27,390 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1159206f1248389897de048517e209, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731259465918 2024-11-10T17:24:27,390 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b85f7637425a40b2a79f90fd9ef3a692, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731259465918 2024-11-10T17:24:27,401 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:27,401 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/d5a081176d004879ba67e01f4fabfc5e is 50, key is test_row_0/B:col10/1731259465918/Put/seqid=0 2024-11-10T17:24:27,410 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:27,411 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/bdae7c845d1944a595bb015411c6dbd9 is 50, key is test_row_0/A:col10/1731259465918/Put/seqid=0 2024-11-10T17:24:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:27,413 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:27,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:24:27,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/85eec6e9240b432a9fa5cfb35c6a9526 is 50, key is test_row_0/A:col10/1731259466173/Put/seqid=0 2024-11-10T17:24:27,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742346_1522 (size=13153) 2024-11-10T17:24:27,462 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/d5a081176d004879ba67e01f4fabfc5e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d5a081176d004879ba67e01f4fabfc5e 2024-11-10T17:24:27,467 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/B of 57c77250ebf57e5a251700719e1bd916 into d5a081176d004879ba67e01f4fabfc5e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:27,467 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:27,468 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/B, priority=13, startTime=1731259467387; duration=0sec 2024-11-10T17:24:27,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:27,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:B 2024-11-10T17:24:27,468 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:27,469 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:27,469 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): 57c77250ebf57e5a251700719e1bd916/C is initiating minor compaction (all files) 2024-11-10T17:24:27,469 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c77250ebf57e5a251700719e1bd916/C in TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,469 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/f82447006c1243f7a7044bca9d50ad76, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/b576d2aafed64fa4b71f379c16477326, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c3cba384090843ca870556fabf186fcc] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp, totalSize=36.8 K 2024-11-10T17:24:27,470 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f82447006c1243f7a7044bca9d50ad76, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731259464507 2024-11-10T17:24:27,470 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b576d2aafed64fa4b71f379c16477326, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731259464703 2024-11-10T17:24:27,470 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c3cba384090843ca870556fabf186fcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731259465918 2024-11-10T17:24:27,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259527458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:27,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259527459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:27,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:27,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:27,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742347_1523 (size=13153) 2024-11-10T17:24:27,486 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/bdae7c845d1944a595bb015411c6dbd9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/bdae7c845d1944a595bb015411c6dbd9 2024-11-10T17:24:27,492 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/A of 57c77250ebf57e5a251700719e1bd916 into bdae7c845d1944a595bb015411c6dbd9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:27,492 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:27,492 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/A, priority=13, startTime=1731259467387; duration=0sec 2024-11-10T17:24:27,492 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:27,492 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:A 2024-11-10T17:24:27,502 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c77250ebf57e5a251700719e1bd916#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:27,503 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/fc35b77b29fb48f78111221d8f046b58 is 50, key is test_row_0/C:col10/1731259465918/Put/seqid=0 2024-11-10T17:24:27,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742348_1524 (size=12301) 2024-11-10T17:24:27,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:27,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:27,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742349_1525 (size=13153) 2024-11-10T17:24:27,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259527581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:27,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259527581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:27,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T17:24:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259527793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259527801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:27,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:27,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:27,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/85eec6e9240b432a9fa5cfb35c6a9526 2024-11-10T17:24:27,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/823ad23c88d342559e55b1624935a643 is 50, key is test_row_0/B:col10/1731259466173/Put/seqid=0 2024-11-10T17:24:27,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742350_1526 (size=12301) 2024-11-10T17:24:27,935 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/fc35b77b29fb48f78111221d8f046b58 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc35b77b29fb48f78111221d8f046b58 2024-11-10T17:24:27,941 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c77250ebf57e5a251700719e1bd916/C of 57c77250ebf57e5a251700719e1bd916 into fc35b77b29fb48f78111221d8f046b58(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:27,942 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:27,942 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., storeName=57c77250ebf57e5a251700719e1bd916/C, priority=13, startTime=1731259467387; duration=0sec 2024-11-10T17:24:27,942 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:27,942 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c77250ebf57e5a251700719e1bd916:C 2024-11-10T17:24:27,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:27,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259528112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:28,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259528112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:28,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T17:24:28,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:28,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:28,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/823ad23c88d342559e55b1624935a643 2024-11-10T17:24:28,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/67188dd7aeed4c1c8345bb3dad416f33 is 50, key is test_row_0/C:col10/1731259466173/Put/seqid=0 2024-11-10T17:24:28,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742351_1527 (size=12301) 2024-11-10T17:24:28,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:28,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:28,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46564 deadline: 1731259528482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:28,489 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46518 deadline: 1731259528521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:28,525 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18335 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:28,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46568 deadline: 1731259528562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:28,574 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in 57c77250ebf57e5a251700719e1bd916 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18381 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:28,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46580 deadline: 1731259528632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:28,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46510 deadline: 1731259528632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 , the old value is region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., hostname=8e3de9160f9f,36421,1731259326553, seqNum=2 because the exception is null or not the one we care about 2024-11-10T17:24:28,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/67188dd7aeed4c1c8345bb3dad416f33 2024-11-10T17:24:28,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/85eec6e9240b432a9fa5cfb35c6a9526 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/85eec6e9240b432a9fa5cfb35c6a9526 2024-11-10T17:24:28,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/85eec6e9240b432a9fa5cfb35c6a9526, entries=150, sequenceid=363, filesize=12.0 K 2024-11-10T17:24:28,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/823ad23c88d342559e55b1624935a643 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/823ad23c88d342559e55b1624935a643 2024-11-10T17:24:28,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/823ad23c88d342559e55b1624935a643, entries=150, sequenceid=363, filesize=12.0 K 2024-11-10T17:24:28,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/67188dd7aeed4c1c8345bb3dad416f33 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/67188dd7aeed4c1c8345bb3dad416f33 2024-11-10T17:24:28,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:28,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/67188dd7aeed4c1c8345bb3dad416f33, entries=150, sequenceid=363, filesize=12.0 K 2024-11-10T17:24:28,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 57c77250ebf57e5a251700719e1bd916 in 1346ms, sequenceid=363, compaction requested=false 2024-11-10T17:24:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:28,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T17:24:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:28,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T17:24:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/6a9e384cbe814ecdb29643e7b5ab0c06 is 50, key is test_row_0/A:col10/1731259467446/Put/seqid=0 2024-11-10T17:24:28,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742352_1528 (size=12301) 2024-11-10T17:24:28,940 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/6a9e384cbe814ecdb29643e7b5ab0c06 2024-11-10T17:24:28,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/827b2def01ba4ab8a1c91d9034a9e4b9 is 50, key is test_row_0/B:col10/1731259467446/Put/seqid=0 2024-11-10T17:24:28,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742353_1529 (size=12301) 2024-11-10T17:24:28,966 INFO [Thread-2016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2016. 2024-11-10T17:24:28,966 DEBUG [Thread-2016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:28,966 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,966 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:28,967 INFO [Thread-2014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2014. 2024-11-10T17:24:28,967 DEBUG [Thread-2014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:28,967 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:28,967 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,968 INFO [Thread-2018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2018. 2024-11-10T17:24:28,968 DEBUG [Thread-2018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:28,968 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,968 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:28,971 INFO [Thread-2010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2010. 2024-11-10T17:24:28,971 DEBUG [Thread-2010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:28,971 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,971 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:28,972 INFO [Thread-2012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2012. 2024-11-10T17:24:28,972 DEBUG [Thread-2012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:28,972 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,972 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:28,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T17:24:29,357 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/827b2def01ba4ab8a1c91d9034a9e4b9 2024-11-10T17:24:29,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/3a368c3302184f6b9ba07fea6a5e4399 is 50, key is test_row_0/C:col10/1731259467446/Put/seqid=0 2024-11-10T17:24:29,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742354_1530 (size=12301) 2024-11-10T17:24:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:29,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. as already flushing 2024-11-10T17:24:29,653 INFO [Thread-2003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2003. 2024-11-10T17:24:29,653 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-11-10T17:24:29,653 DEBUG [Thread-2003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:29,653 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:29,653 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:29,653 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:29,653 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:29,653 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:29,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:29,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:29,766 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/3a368c3302184f6b9ba07fea6a5e4399 2024-11-10T17:24:29,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/6a9e384cbe814ecdb29643e7b5ab0c06 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6a9e384cbe814ecdb29643e7b5ab0c06 2024-11-10T17:24:29,772 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6a9e384cbe814ecdb29643e7b5ab0c06, entries=150, sequenceid=376, filesize=12.0 K 2024-11-10T17:24:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/827b2def01ba4ab8a1c91d9034a9e4b9 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/827b2def01ba4ab8a1c91d9034a9e4b9 2024-11-10T17:24:29,775 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/827b2def01ba4ab8a1c91d9034a9e4b9, entries=150, sequenceid=376, filesize=12.0 K 2024-11-10T17:24:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/3a368c3302184f6b9ba07fea6a5e4399 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3a368c3302184f6b9ba07fea6a5e4399 2024-11-10T17:24:29,778 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3a368c3302184f6b9ba07fea6a5e4399, entries=150, sequenceid=376, filesize=12.0 K 2024-11-10T17:24:29,779 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=13.42 KB/13740 for 57c77250ebf57e5a251700719e1bd916 in 870ms, sequenceid=376, compaction requested=true 2024-11-10T17:24:29,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 57c77250ebf57e5a251700719e1bd916: 2024-11-10T17:24:29,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:29,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-10T17:24:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-10T17:24:29,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T17:24:29,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7300 sec 2024-11-10T17:24:29,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.7370 sec 2024-11-10T17:24:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T17:24:31,222 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:34,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T17:24:38,533 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-10T17:24:38,533 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:38,533 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,534 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:38,542 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-11-10T17:24:38,542 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:38,542 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,543 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:38,623 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-11-10T17:24:38,623 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:24:38,623 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,623 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 193 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1273 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3818 rows 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1282 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3845 rows 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1273 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3813 rows 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1274 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3818 rows 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1281 2024-11-10T17:24:38,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3839 rows 2024-11-10T17:24:38,624 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:24:38,624 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:24:38,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:38,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:24:38,625 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T17:24:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T17:24:38,627 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259478627"}]},"ts":"1731259478627"} 2024-11-10T17:24:38,628 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T17:24:38,629 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T17:24:38,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:24:38,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, UNASSIGN}] 2024-11-10T17:24:38,631 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, UNASSIGN 2024-11-10T17:24:38,631 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=57c77250ebf57e5a251700719e1bd916, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:38,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, UNASSIGN because future has completed 2024-11-10T17:24:38,633 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:24:38,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:24:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T17:24:38,785 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:38,785 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:24:38,785 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing 57c77250ebf57e5a251700719e1bd916, disabling compactions & flushes 2024-11-10T17:24:38,785 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:38,785 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:38,785 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. after waiting 0 ms 2024-11-10T17:24:38,785 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:38,785 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing 57c77250ebf57e5a251700719e1bd916 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T17:24:38,788 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/0878ac1849ab4fc9aff9db825dcdacca is 50, key is test_row_0/A:col10/1731259478541/Put/seqid=0 2024-11-10T17:24:38,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742355_1531 (size=12301) 2024-11-10T17:24:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T17:24:39,192 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/0878ac1849ab4fc9aff9db825dcdacca 2024-11-10T17:24:39,197 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/316b2fd8939542e6991add443ad7cc2f is 50, key is test_row_0/B:col10/1731259478541/Put/seqid=0 2024-11-10T17:24:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742356_1532 (size=12301) 2024-11-10T17:24:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T17:24:39,601 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/316b2fd8939542e6991add443ad7cc2f 2024-11-10T17:24:39,606 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c9a58b0f30824cc19b19567055006dba is 50, key is test_row_0/C:col10/1731259478541/Put/seqid=0 2024-11-10T17:24:39,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742357_1533 (size=12301) 2024-11-10T17:24:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T17:24:40,010 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c9a58b0f30824cc19b19567055006dba 2024-11-10T17:24:40,013 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/A/0878ac1849ab4fc9aff9db825dcdacca as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/0878ac1849ab4fc9aff9db825dcdacca 2024-11-10T17:24:40,015 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/0878ac1849ab4fc9aff9db825dcdacca, entries=150, sequenceid=384, filesize=12.0 K 2024-11-10T17:24:40,016 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/B/316b2fd8939542e6991add443ad7cc2f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/316b2fd8939542e6991add443ad7cc2f 2024-11-10T17:24:40,018 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/316b2fd8939542e6991add443ad7cc2f, entries=150, sequenceid=384, filesize=12.0 K 2024-11-10T17:24:40,019 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/.tmp/C/c9a58b0f30824cc19b19567055006dba as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c9a58b0f30824cc19b19567055006dba 2024-11-10T17:24:40,021 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c9a58b0f30824cc19b19567055006dba, entries=150, sequenceid=384, filesize=12.0 K 2024-11-10T17:24:40,022 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 57c77250ebf57e5a251700719e1bd916 in 1237ms, sequenceid=384, compaction requested=true 2024-11-10T17:24:40,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/08f0366d5ad140a495fc4f0cd4478488, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/754cbc31788f4b3b96df59f1b4ae55e6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6febe9343eb14a41af383e4dca916a49, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5dce24b2ca6f4fa7b71a29db2aab604b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7fba275a67c540b7a6dd76372daf5985, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7f0e5c293fd64a6c9ad90146fdca597a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/49a844ddcf0540af8af24bd46040eed0, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/815831b44364439d921ab6c1949f6b19, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/4c8270b54d6845cc9f51e497bf12f8f0, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/ab60e4501904435898a05b32a26c3975, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/f61baec694604f33bd299ac59a29b0cb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d5e3efde9a2c440889059604bcfa0804, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/94e8a893d58348538bbba17ba28dd505, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/30356236ede0498a83a4e10633c402b5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d816149303364f2f92cf7c126f7f03e4, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5063feb09d5a485a9f95f5df2b85b5e7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/b99b92f4134f4c609d8bba68c7470a29, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/e2ff4f46a7c84276b0c64a2daaa29aef, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/623f13c4b6c34ef383fe841d9a807066, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5f8fe254a5f34ed1897882f772336e96, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d4452f9668364a10b82c12fb8af8a08f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/c94c288e66cf4ceebcb467e8d50c7e53, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/3c0c1aedd8234d4b992f3dbc22189eec, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5be05660a9424145b5cafcf6a965ea06, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/dd1159206f1248389897de048517e209] to archive 2024-11-10T17:24:40,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:24:40,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/08f0366d5ad140a495fc4f0cd4478488 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/08f0366d5ad140a495fc4f0cd4478488 2024-11-10T17:24:40,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/754cbc31788f4b3b96df59f1b4ae55e6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/754cbc31788f4b3b96df59f1b4ae55e6 2024-11-10T17:24:40,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6febe9343eb14a41af383e4dca916a49 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6febe9343eb14a41af383e4dca916a49 2024-11-10T17:24:40,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5dce24b2ca6f4fa7b71a29db2aab604b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5dce24b2ca6f4fa7b71a29db2aab604b 2024-11-10T17:24:40,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7fba275a67c540b7a6dd76372daf5985 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7fba275a67c540b7a6dd76372daf5985 2024-11-10T17:24:40,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7f0e5c293fd64a6c9ad90146fdca597a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/7f0e5c293fd64a6c9ad90146fdca597a 2024-11-10T17:24:40,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/49a844ddcf0540af8af24bd46040eed0 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/49a844ddcf0540af8af24bd46040eed0 2024-11-10T17:24:40,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/815831b44364439d921ab6c1949f6b19 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/815831b44364439d921ab6c1949f6b19 2024-11-10T17:24:40,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/4c8270b54d6845cc9f51e497bf12f8f0 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/4c8270b54d6845cc9f51e497bf12f8f0 2024-11-10T17:24:40,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/ab60e4501904435898a05b32a26c3975 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/ab60e4501904435898a05b32a26c3975 2024-11-10T17:24:40,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/f61baec694604f33bd299ac59a29b0cb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/f61baec694604f33bd299ac59a29b0cb 2024-11-10T17:24:40,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d5e3efde9a2c440889059604bcfa0804 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d5e3efde9a2c440889059604bcfa0804 2024-11-10T17:24:40,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/94e8a893d58348538bbba17ba28dd505 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/94e8a893d58348538bbba17ba28dd505 2024-11-10T17:24:40,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/30356236ede0498a83a4e10633c402b5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/30356236ede0498a83a4e10633c402b5 2024-11-10T17:24:40,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d816149303364f2f92cf7c126f7f03e4 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d816149303364f2f92cf7c126f7f03e4 2024-11-10T17:24:40,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5063feb09d5a485a9f95f5df2b85b5e7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5063feb09d5a485a9f95f5df2b85b5e7 2024-11-10T17:24:40,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/b99b92f4134f4c609d8bba68c7470a29 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/b99b92f4134f4c609d8bba68c7470a29 2024-11-10T17:24:40,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/e2ff4f46a7c84276b0c64a2daaa29aef to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/e2ff4f46a7c84276b0c64a2daaa29aef 2024-11-10T17:24:40,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/623f13c4b6c34ef383fe841d9a807066 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/623f13c4b6c34ef383fe841d9a807066 2024-11-10T17:24:40,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5f8fe254a5f34ed1897882f772336e96 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5f8fe254a5f34ed1897882f772336e96 2024-11-10T17:24:40,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d4452f9668364a10b82c12fb8af8a08f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/d4452f9668364a10b82c12fb8af8a08f 2024-11-10T17:24:40,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/c94c288e66cf4ceebcb467e8d50c7e53 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/c94c288e66cf4ceebcb467e8d50c7e53 2024-11-10T17:24:40,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/3c0c1aedd8234d4b992f3dbc22189eec to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/3c0c1aedd8234d4b992f3dbc22189eec 2024-11-10T17:24:40,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5be05660a9424145b5cafcf6a965ea06 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/5be05660a9424145b5cafcf6a965ea06 2024-11-10T17:24:40,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/dd1159206f1248389897de048517e209 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/dd1159206f1248389897de048517e209 2024-11-10T17:24:40,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/77cbf86083ab4376b70b0831f308bfa5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7731cb58800840b390095ee5193b2630, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/dd65ea156f984512a011529177db474b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d372bee0e78a416f92944a8a668b7b0f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/6ee48826f4ef4bd9937cd2cc6a7c6876, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c28eb65e99024920b87c136db67565b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4ceaf19eafc94ab98dde689c9f8e72aa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/51ceb7c149a341e3ad68368b6ff24cd7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3c153e9c6a9249caa5c75e7a9548a1ae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/bde23ecee19d476795dc63035eafe093, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00ca6c25e2e94ac480e2379cb30d4513, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/5c9e21356c514fcf8f48ad1fee3ab088, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/17e357716a4843849bad930ee287cbae, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/434a794f99bd4fdcaef22bfeb0eb41de, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/1027c23e822f452f919dcf5bc7252b15, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3516ac0fa009400c901e48a66b201370, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/288d900ed00c43de955137f226684c99, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4c591835680e40c78818d3c5754daa97, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00819ce45aad4915a7d8ea083fded23c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/9d25a52afebc44719fbb8a1ae89a4457, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3f15447ee40e4500995f6e3fc4a1378f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7b2a0232a91e46aab985e5455500b43b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c8c3dba820844d07bc9bb3485c3bfcb7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/98ab2194196b46f19472a86ea4544943, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/b85f7637425a40b2a79f90fd9ef3a692] to archive 2024-11-10T17:24:40,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:24:40,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/77cbf86083ab4376b70b0831f308bfa5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/77cbf86083ab4376b70b0831f308bfa5 2024-11-10T17:24:40,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7731cb58800840b390095ee5193b2630 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7731cb58800840b390095ee5193b2630 2024-11-10T17:24:40,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/dd65ea156f984512a011529177db474b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/dd65ea156f984512a011529177db474b 2024-11-10T17:24:40,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d372bee0e78a416f92944a8a668b7b0f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d372bee0e78a416f92944a8a668b7b0f 2024-11-10T17:24:40,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/6ee48826f4ef4bd9937cd2cc6a7c6876 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/6ee48826f4ef4bd9937cd2cc6a7c6876 2024-11-10T17:24:40,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c28eb65e99024920b87c136db67565b6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c28eb65e99024920b87c136db67565b6 2024-11-10T17:24:40,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4ceaf19eafc94ab98dde689c9f8e72aa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4ceaf19eafc94ab98dde689c9f8e72aa 2024-11-10T17:24:40,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/51ceb7c149a341e3ad68368b6ff24cd7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/51ceb7c149a341e3ad68368b6ff24cd7 2024-11-10T17:24:40,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3c153e9c6a9249caa5c75e7a9548a1ae to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3c153e9c6a9249caa5c75e7a9548a1ae 2024-11-10T17:24:40,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/bde23ecee19d476795dc63035eafe093 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/bde23ecee19d476795dc63035eafe093 2024-11-10T17:24:40,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00ca6c25e2e94ac480e2379cb30d4513 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00ca6c25e2e94ac480e2379cb30d4513 2024-11-10T17:24:40,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/5c9e21356c514fcf8f48ad1fee3ab088 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/5c9e21356c514fcf8f48ad1fee3ab088 2024-11-10T17:24:40,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/17e357716a4843849bad930ee287cbae to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/17e357716a4843849bad930ee287cbae 2024-11-10T17:24:40,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/434a794f99bd4fdcaef22bfeb0eb41de to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/434a794f99bd4fdcaef22bfeb0eb41de 2024-11-10T17:24:40,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/1027c23e822f452f919dcf5bc7252b15 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/1027c23e822f452f919dcf5bc7252b15 2024-11-10T17:24:40,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3516ac0fa009400c901e48a66b201370 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3516ac0fa009400c901e48a66b201370 2024-11-10T17:24:40,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/288d900ed00c43de955137f226684c99 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/288d900ed00c43de955137f226684c99 2024-11-10T17:24:40,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4c591835680e40c78818d3c5754daa97 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/4c591835680e40c78818d3c5754daa97 2024-11-10T17:24:40,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00819ce45aad4915a7d8ea083fded23c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/00819ce45aad4915a7d8ea083fded23c 2024-11-10T17:24:40,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/9d25a52afebc44719fbb8a1ae89a4457 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/9d25a52afebc44719fbb8a1ae89a4457 2024-11-10T17:24:40,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3f15447ee40e4500995f6e3fc4a1378f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/3f15447ee40e4500995f6e3fc4a1378f 2024-11-10T17:24:40,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7b2a0232a91e46aab985e5455500b43b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/7b2a0232a91e46aab985e5455500b43b 2024-11-10T17:24:40,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c8c3dba820844d07bc9bb3485c3bfcb7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/c8c3dba820844d07bc9bb3485c3bfcb7 2024-11-10T17:24:40,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/98ab2194196b46f19472a86ea4544943 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/98ab2194196b46f19472a86ea4544943 2024-11-10T17:24:40,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/b85f7637425a40b2a79f90fd9ef3a692 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/b85f7637425a40b2a79f90fd9ef3a692 2024-11-10T17:24:40,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/e4c1581460204a50a927986ff7b74d4e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/89bc591a19fd4c1fa3fe97106699db11, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/81aff07caf2a4bf3ba94b2702866c8e6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6f8c166d79854809a21c0e0cde1ca46c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/bef92cda2f83419bbac5b65251fc7792, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/114f476687374a7e924d1e1ee4702c71, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/80e8bf604a504033b172598a653b0cc6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/1ef4189c77fb4cf6b3c6be0ec13f5d52, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/21d31c687dd544eca545c21928cbf447, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6befb8e72a43425d8c555e570dcb5598, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/a2874192be9644c2a7e60bc751c3673c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c578df881084497592ef54bf4e041fb3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5ef81b48c2b4428a8f021ae4235558de, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3b7e550c6077438881cb8e1b8b85c177, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5418c8e6a4e449b18821dd6005b20aaa, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/60652cae52a3408bb6f832fbb05bef3d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/12583befee454049bfbc81071b8dc39f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/63550f6ba45b4eac9299cb94d5b9658a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc68de03296f4234bb86f850824a8d11, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8a46c399d63449a2ae1b2925e7eb1fcf, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/9b81e63970254d02aaa19b18de3c906d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/f82447006c1243f7a7044bca9d50ad76, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8eae05bbcfc1463580c594fd868efc7e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/b576d2aafed64fa4b71f379c16477326, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c3cba384090843ca870556fabf186fcc] to archive 2024-11-10T17:24:40,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:24:40,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/e4c1581460204a50a927986ff7b74d4e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/e4c1581460204a50a927986ff7b74d4e 2024-11-10T17:24:40,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/89bc591a19fd4c1fa3fe97106699db11 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/89bc591a19fd4c1fa3fe97106699db11 2024-11-10T17:24:40,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/81aff07caf2a4bf3ba94b2702866c8e6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/81aff07caf2a4bf3ba94b2702866c8e6 2024-11-10T17:24:40,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6f8c166d79854809a21c0e0cde1ca46c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6f8c166d79854809a21c0e0cde1ca46c 2024-11-10T17:24:40,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/bef92cda2f83419bbac5b65251fc7792 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/bef92cda2f83419bbac5b65251fc7792 2024-11-10T17:24:40,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/114f476687374a7e924d1e1ee4702c71 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/114f476687374a7e924d1e1ee4702c71 2024-11-10T17:24:40,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/80e8bf604a504033b172598a653b0cc6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/80e8bf604a504033b172598a653b0cc6 2024-11-10T17:24:40,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/1ef4189c77fb4cf6b3c6be0ec13f5d52 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/1ef4189c77fb4cf6b3c6be0ec13f5d52 2024-11-10T17:24:40,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/21d31c687dd544eca545c21928cbf447 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/21d31c687dd544eca545c21928cbf447 2024-11-10T17:24:40,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6befb8e72a43425d8c555e570dcb5598 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/6befb8e72a43425d8c555e570dcb5598 2024-11-10T17:24:40,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/a2874192be9644c2a7e60bc751c3673c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/a2874192be9644c2a7e60bc751c3673c 2024-11-10T17:24:40,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c578df881084497592ef54bf4e041fb3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c578df881084497592ef54bf4e041fb3 2024-11-10T17:24:40,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5ef81b48c2b4428a8f021ae4235558de to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5ef81b48c2b4428a8f021ae4235558de 2024-11-10T17:24:40,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3b7e550c6077438881cb8e1b8b85c177 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3b7e550c6077438881cb8e1b8b85c177 2024-11-10T17:24:40,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5418c8e6a4e449b18821dd6005b20aaa to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/5418c8e6a4e449b18821dd6005b20aaa 2024-11-10T17:24:40,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/60652cae52a3408bb6f832fbb05bef3d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/60652cae52a3408bb6f832fbb05bef3d 2024-11-10T17:24:40,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/12583befee454049bfbc81071b8dc39f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/12583befee454049bfbc81071b8dc39f 2024-11-10T17:24:40,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/63550f6ba45b4eac9299cb94d5b9658a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/63550f6ba45b4eac9299cb94d5b9658a 2024-11-10T17:24:40,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc68de03296f4234bb86f850824a8d11 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc68de03296f4234bb86f850824a8d11 2024-11-10T17:24:40,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8a46c399d63449a2ae1b2925e7eb1fcf to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8a46c399d63449a2ae1b2925e7eb1fcf 2024-11-10T17:24:40,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/9b81e63970254d02aaa19b18de3c906d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/9b81e63970254d02aaa19b18de3c906d 2024-11-10T17:24:40,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/f82447006c1243f7a7044bca9d50ad76 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/f82447006c1243f7a7044bca9d50ad76 2024-11-10T17:24:40,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8eae05bbcfc1463580c594fd868efc7e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/8eae05bbcfc1463580c594fd868efc7e 2024-11-10T17:24:40,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/b576d2aafed64fa4b71f379c16477326 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/b576d2aafed64fa4b71f379c16477326 2024-11-10T17:24:40,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c3cba384090843ca870556fabf186fcc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c3cba384090843ca870556fabf186fcc 2024-11-10T17:24:40,083 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/recovered.edits/387.seqid, newMaxSeqId=387, maxSeqId=1 2024-11-10T17:24:40,084 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. 2024-11-10T17:24:40,084 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for 57c77250ebf57e5a251700719e1bd916: Waiting for close lock at 1731259478785Running coprocessor pre-close hooks at 1731259478785Disabling compacts and flushes for region at 1731259478785Disabling writes for close at 1731259478785Obtaining lock to block concurrent updates at 1731259478785Preparing flush snapshotting stores in 57c77250ebf57e5a251700719e1bd916 at 1731259478785Finished memstore snapshotting TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731259478785Flushing stores of TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916. at 1731259478786 (+1 ms)Flushing 57c77250ebf57e5a251700719e1bd916/A: creating writer at 1731259478786Flushing 57c77250ebf57e5a251700719e1bd916/A: appending metadata at 1731259478788 (+2 ms)Flushing 57c77250ebf57e5a251700719e1bd916/A: closing flushed file at 1731259478788Flushing 57c77250ebf57e5a251700719e1bd916/B: creating writer at 1731259479195 (+407 ms)Flushing 57c77250ebf57e5a251700719e1bd916/B: appending metadata at 1731259479197 (+2 ms)Flushing 57c77250ebf57e5a251700719e1bd916/B: closing flushed file at 1731259479197Flushing 57c77250ebf57e5a251700719e1bd916/C: creating writer at 1731259479604 (+407 ms)Flushing 57c77250ebf57e5a251700719e1bd916/C: appending metadata at 1731259479606 (+2 ms)Flushing 57c77250ebf57e5a251700719e1bd916/C: closing flushed file at 1731259479606Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a5ec4f0: reopening flushed file at 1731259480012 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e1f56b: reopening flushed file at 1731259480015 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44271705: reopening flushed file at 1731259480018 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 57c77250ebf57e5a251700719e1bd916 in 1237ms, sequenceid=384, compaction requested=true at 1731259480022 (+4 ms)Writing region close event to WAL at 1731259480081 (+59 ms)Running coprocessor post-close hooks at 1731259480084 (+3 ms)Closed at 1731259480084 2024-11-10T17:24:40,085 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed 57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:40,086 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=57c77250ebf57e5a251700719e1bd916, regionState=CLOSED 2024-11-10T17:24:40,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:24:40,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T17:24:40,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure 57c77250ebf57e5a251700719e1bd916, server=8e3de9160f9f,36421,1731259326553 in 1.4550 sec 2024-11-10T17:24:40,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-10T17:24:40,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c77250ebf57e5a251700719e1bd916, UNASSIGN in 1.4590 sec 2024-11-10T17:24:40,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T17:24:40,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4610 sec 2024-11-10T17:24:40,092 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259480092"}]},"ts":"1731259480092"} 2024-11-10T17:24:40,093 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T17:24:40,093 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T17:24:40,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4690 sec 2024-11-10T17:24:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T17:24:40,772 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:40,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T17:24:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:40,773 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T17:24:40,774 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:40,776 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:40,777 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/recovered.edits] 2024-11-10T17:24:40,779 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/0878ac1849ab4fc9aff9db825dcdacca to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/0878ac1849ab4fc9aff9db825dcdacca 2024-11-10T17:24:40,780 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6a9e384cbe814ecdb29643e7b5ab0c06 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/6a9e384cbe814ecdb29643e7b5ab0c06 2024-11-10T17:24:40,781 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/85eec6e9240b432a9fa5cfb35c6a9526 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/85eec6e9240b432a9fa5cfb35c6a9526 2024-11-10T17:24:40,782 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/bdae7c845d1944a595bb015411c6dbd9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/A/bdae7c845d1944a595bb015411c6dbd9 2024-11-10T17:24:40,784 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/316b2fd8939542e6991add443ad7cc2f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/316b2fd8939542e6991add443ad7cc2f 2024-11-10T17:24:40,785 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/823ad23c88d342559e55b1624935a643 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/823ad23c88d342559e55b1624935a643 2024-11-10T17:24:40,785 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/827b2def01ba4ab8a1c91d9034a9e4b9 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/827b2def01ba4ab8a1c91d9034a9e4b9 2024-11-10T17:24:40,786 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d5a081176d004879ba67e01f4fabfc5e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/B/d5a081176d004879ba67e01f4fabfc5e 2024-11-10T17:24:40,788 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3a368c3302184f6b9ba07fea6a5e4399 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/3a368c3302184f6b9ba07fea6a5e4399 2024-11-10T17:24:40,789 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/67188dd7aeed4c1c8345bb3dad416f33 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/67188dd7aeed4c1c8345bb3dad416f33 2024-11-10T17:24:40,790 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c9a58b0f30824cc19b19567055006dba to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/c9a58b0f30824cc19b19567055006dba 2024-11-10T17:24:40,790 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc35b77b29fb48f78111221d8f046b58 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/C/fc35b77b29fb48f78111221d8f046b58 2024-11-10T17:24:40,792 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/recovered.edits/387.seqid to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916/recovered.edits/387.seqid 2024-11-10T17:24:40,793 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/57c77250ebf57e5a251700719e1bd916 2024-11-10T17:24:40,793 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T17:24:40,794 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:40,796 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T17:24:40,798 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T17:24:40,798 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:40,798 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T17:24:40,799 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731259480798"}]},"ts":"9223372036854775807"} 2024-11-10T17:24:40,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T17:24:40,800 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 57c77250ebf57e5a251700719e1bd916, NAME => 'TestAcidGuarantees,,1731259446768.57c77250ebf57e5a251700719e1bd916.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T17:24:40,800 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T17:24:40,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731259480800"}]},"ts":"9223372036854775807"} 2024-11-10T17:24:40,801 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T17:24:40,802 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:40,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-10T17:24:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T17:24:40,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T17:24:40,882 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:40,890 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 237), OpenFileDescriptor=441 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 393), ProcessCount=11 (was 11), AvailableMemoryMB=5859 (was 6030) 2024-11-10T17:24:40,898 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=5859 2024-11-10T17:24:40,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:24:40,900 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:24:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:40,901 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T17:24:40,901 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:40,901 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-11-10T17:24:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T17:24:40,902 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T17:24:40,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742358_1534 (size=959) 2024-11-10T17:24:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T17:24:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T17:24:41,308 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a 2024-11-10T17:24:41,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742359_1535 (size=53) 2024-11-10T17:24:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T17:24:41,713 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:24:41,713 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing da5a26d0da1375a506f6b0241d015eea, disabling compactions & flushes 2024-11-10T17:24:41,713 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:41,713 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:41,713 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. after waiting 0 ms 2024-11-10T17:24:41,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:41,714 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:41,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for da5a26d0da1375a506f6b0241d015eea: Waiting for close lock at 1731259481713Disabling compacts and flushes for region at 1731259481713Disabling writes for close at 1731259481713Writing region close event to WAL at 1731259481714 (+1 ms)Closed at 1731259481714 2024-11-10T17:24:41,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T17:24:41,715 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731259481715"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731259481715"}]},"ts":"1731259481715"} 2024-11-10T17:24:41,716 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T17:24:41,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T17:24:41,717 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259481717"}]},"ts":"1731259481717"} 2024-11-10T17:24:41,718 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T17:24:41,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, ASSIGN}] 2024-11-10T17:24:41,719 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, ASSIGN 2024-11-10T17:24:41,719 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, ASSIGN; state=OFFLINE, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=false 2024-11-10T17:24:41,870 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:41,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, ASSIGN because future has completed 2024-11-10T17:24:41,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:24:42,026 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:42,026 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:24:42,026 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,026 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:24:42,026 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,026 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,027 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,029 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5a26d0da1375a506f6b0241d015eea columnFamilyName A 2024-11-10T17:24:42,029 DEBUG [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:42,029 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(327): Store=da5a26d0da1375a506f6b0241d015eea/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:42,029 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,030 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5a26d0da1375a506f6b0241d015eea columnFamilyName B 2024-11-10T17:24:42,030 DEBUG [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:42,030 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(327): Store=da5a26d0da1375a506f6b0241d015eea/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:42,030 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,031 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5a26d0da1375a506f6b0241d015eea columnFamilyName C 2024-11-10T17:24:42,031 DEBUG [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:42,031 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(327): Store=da5a26d0da1375a506f6b0241d015eea/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:42,031 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,032 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,032 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,033 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,033 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,033 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:24:42,034 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,036 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T17:24:42,036 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened da5a26d0da1375a506f6b0241d015eea; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62560174, jitterRate=-0.0677807629108429}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:24:42,036 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:42,037 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for da5a26d0da1375a506f6b0241d015eea: Running coprocessor pre-open hook at 1731259482026Writing region info on filesystem at 1731259482026Initializing all the Stores at 1731259482027 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259482027Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259482027Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259482027Cleaning up temporary data from old regions at 1731259482033 (+6 ms)Running coprocessor post-open hooks at 1731259482036 (+3 ms)Region opened successfully at 1731259482036 2024-11-10T17:24:42,037 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., pid=132, masterSystemTime=1731259482023 2024-11-10T17:24:42,038 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:42,039 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:42,039 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:42,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:24:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T17:24:42,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T17:24:42,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 in 169 msec 2024-11-10T17:24:42,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-10T17:24:42,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, ASSIGN in 324 msec 2024-11-10T17:24:42,044 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T17:24:42,044 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259482044"}]},"ts":"1731259482044"} 2024-11-10T17:24:42,045 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T17:24:42,045 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T17:24:42,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-10T17:24:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T17:24:43,062 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:43,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,064 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c8ae63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,066 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765137b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,067 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:43,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:43,069 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T17:24:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,40007,1731259325831 2024-11-10T17:24:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b606 2024-11-10T17:24:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T17:24:43,070 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T17:24:43,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T17:24:43,072 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T17:24:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T17:24:43,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742360_1536 (size=995) 2024-11-10T17:24:43,481 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T17:24:43,481 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T17:24:43,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:24:43,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, REOPEN/MOVE}] 2024-11-10T17:24:43,486 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, REOPEN/MOVE 2024-11-10T17:24:43,486 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:43,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, REOPEN/MOVE because future has completed 2024-11-10T17:24:43,488 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:24:43,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:24:43,640 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,640 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:24:43,640 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing da5a26d0da1375a506f6b0241d015eea, disabling compactions & flushes 2024-11-10T17:24:43,640 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:43,640 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:43,640 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. after waiting 0 ms 2024-11-10T17:24:43,640 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:43,643 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T17:24:43,643 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:43,644 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for da5a26d0da1375a506f6b0241d015eea: Waiting for close lock at 1731259483640Running coprocessor pre-close hooks at 1731259483640Disabling compacts and flushes for region at 1731259483640Disabling writes for close at 1731259483640Writing region close event to WAL at 1731259483640Running coprocessor post-close hooks at 1731259483643 (+3 ms)Closed at 1731259483643 2024-11-10T17:24:43,644 WARN [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionServer(3299): Not adding moved region record: da5a26d0da1375a506f6b0241d015eea to self. 2024-11-10T17:24:43,645 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=CLOSED 2024-11-10T17:24:43,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:24:43,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T17:24:43,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 in 159 msec 2024-11-10T17:24:43,649 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, REOPEN/MOVE; state=CLOSED, location=8e3de9160f9f,36421,1731259326553; forceNewPlan=false, retain=true 2024-11-10T17:24:43,799 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=OPENING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:43,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, REOPEN/MOVE because future has completed 2024-11-10T17:24:43,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:24:43,955 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:43,955 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7752): Opening region: {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} 2024-11-10T17:24:43,955 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,955 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T17:24:43,956 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7794): checking encryption for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,956 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7797): checking classloading for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,957 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,957 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5a26d0da1375a506f6b0241d015eea columnFamilyName A 2024-11-10T17:24:43,958 DEBUG [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:43,959 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(327): Store=da5a26d0da1375a506f6b0241d015eea/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:43,959 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,959 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5a26d0da1375a506f6b0241d015eea columnFamilyName B 2024-11-10T17:24:43,959 DEBUG [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:43,960 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(327): Store=da5a26d0da1375a506f6b0241d015eea/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:43,960 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,960 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5a26d0da1375a506f6b0241d015eea columnFamilyName C 2024-11-10T17:24:43,960 DEBUG [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:43,961 INFO [StoreOpener-da5a26d0da1375a506f6b0241d015eea-1 {}] regionserver.HStore(327): Store=da5a26d0da1375a506f6b0241d015eea/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T17:24:43,961 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1038): replaying wal for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,961 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,962 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,963 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1048): stopping wal replay for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,963 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1060): Cleaning up temporary data for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,963 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T17:24:43,964 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1093): writing seq id for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,964 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1114): Opened da5a26d0da1375a506f6b0241d015eea; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65542266, jitterRate=-0.02334412932395935}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T17:24:43,964 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:43,965 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1006): Region open journal for da5a26d0da1375a506f6b0241d015eea: Running coprocessor pre-open hook at 1731259483956Writing region info on filesystem at 1731259483956Initializing all the Stores at 1731259483956Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259483956Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259483956Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731259483956Cleaning up temporary data from old regions at 1731259483963 (+7 ms)Running coprocessor post-open hooks at 1731259483964 (+1 ms)Region opened successfully at 1731259483965 (+1 ms) 2024-11-10T17:24:43,966 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., pid=137, masterSystemTime=1731259483952 2024-11-10T17:24:43,967 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:43,967 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:43,968 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=OPEN, openSeqNum=5, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:43,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:24:43,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=135 2024-11-10T17:24:43,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 in 168 msec 2024-11-10T17:24:43,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-10T17:24:43,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, REOPEN/MOVE in 486 msec 2024-11-10T17:24:43,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T17:24:43,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-10T17:24:43,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-11-10T17:24:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T17:24:43,976 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:43,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b26ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f48663a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,980 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc19cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7543a232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26388abd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3c64b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,986 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a2e7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65df9b04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,989 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13982de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb3f7f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,991 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12123472, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,992 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3ae122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,994 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a537834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,995 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692e1747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,996 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7087edd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:43,998 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb8865e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:43,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:43,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:43,999 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8da3d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:44,000 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35cdb022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:44,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:44,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,40007,-1 for getting cluster id 2024-11-10T17:24:44,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T17:24:44,002 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b79f36ae-4c52-42ea-8782-f05a267fe934' 2024-11-10T17:24:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T17:24:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b79f36ae-4c52-42ea-8782-f05a267fe934" 2024-11-10T17:24:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4ec636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,40007,-1] 2024-11-10T17:24:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T17:24:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:24:44,003 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T17:24:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a30cd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T17:24:44,004 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,004 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,004 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,004 DEBUG [Thread-2392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,004 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,004 DEBUG [Thread-2394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,005 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,005 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,005 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,005 DEBUG [Thread-2398 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,005 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T17:24:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,006 DEBUG [Thread-2398 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:44,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,008 DEBUG [Thread-2396 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,36421,1731259326553, seqNum=-1] 2024-11-10T17:24:44,008 DEBUG [Thread-2396 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T17:24:44,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-10T17:24:44,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T17:24:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,010 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,011 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:44,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T17:24:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5] 2024-11-10T17:24:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:44,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:44,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:24:44,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259544040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259544040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a8d401c8f3824010afd3d67ef6eabe4a_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259484016/Put/seqid=0 2024-11-10T17:24:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259544045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259544045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259544046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742361_1537 (size=17034) 2024-11-10T17:24:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T17:24:44,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259544151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259544152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259544162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259544162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259544162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:44,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:44,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T17:24:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259544362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259544362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259544372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259544372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259544372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,456 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:44,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a8d401c8f3824010afd3d67ef6eabe4a_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a8d401c8f3824010afd3d67ef6eabe4a_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:44,460 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c5e7cb6c106f4d41bb8667a994fc87f6, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:44,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c5e7cb6c106f4d41bb8667a994fc87f6 is 175, key is test_row_0/A:col10/1731259484016/Put/seqid=0 2024-11-10T17:24:44,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742362_1538 (size=48139) 2024-11-10T17:24:44,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T17:24:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259544672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259544672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259544682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259544682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259544682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:44,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,865 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c5e7cb6c106f4d41bb8667a994fc87f6 2024-11-10T17:24:44,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/dd4c62422bfa4765a7abb88d79dafca1 is 50, key is test_row_0/B:col10/1731259484016/Put/seqid=0 2024-11-10T17:24:44,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742363_1539 (size=12001) 2024-11-10T17:24:44,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:44,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:45,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:45,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:45,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:45,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:45,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:45,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T17:24:45,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259545181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:45,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:45,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259545182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259545192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:45,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259545192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:45,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259545192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:45,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:45,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:45,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:45,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:45,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:45,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:45,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/dd4c62422bfa4765a7abb88d79dafca1 2024-11-10T17:24:45,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/b3c29464ff584fbe8afdd7ab8c1a0447 is 50, key is test_row_0/C:col10/1731259484016/Put/seqid=0 2024-11-10T17:24:45,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742364_1540 (size=12001) 2024-11-10T17:24:45,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/b3c29464ff584fbe8afdd7ab8c1a0447 2024-11-10T17:24:45,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c5e7cb6c106f4d41bb8667a994fc87f6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c5e7cb6c106f4d41bb8667a994fc87f6 2024-11-10T17:24:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c5e7cb6c106f4d41bb8667a994fc87f6, entries=250, sequenceid=16, filesize=47.0 K 2024-11-10T17:24:45,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/dd4c62422bfa4765a7abb88d79dafca1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/dd4c62422bfa4765a7abb88d79dafca1 2024-11-10T17:24:45,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/dd4c62422bfa4765a7abb88d79dafca1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T17:24:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/b3c29464ff584fbe8afdd7ab8c1a0447 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b3c29464ff584fbe8afdd7ab8c1a0447 2024-11-10T17:24:45,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b3c29464ff584fbe8afdd7ab8c1a0447, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T17:24:45,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for da5a26d0da1375a506f6b0241d015eea in 1318ms, sequenceid=16, compaction requested=false 2024-11-10T17:24:45,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:45,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T17:24:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:45,388 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:24:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103768c29d6d0b422f95e366b32516ac4c_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259484044/Put/seqid=0 2024-11-10T17:24:45,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742365_1541 (size=12154) 2024-11-10T17:24:45,604 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T17:24:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:45,802 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103768c29d6d0b422f95e366b32516ac4c_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103768c29d6d0b422f95e366b32516ac4c_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:45,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/315bed0e0d7748098270fe997b92211c, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/315bed0e0d7748098270fe997b92211c is 175, key is test_row_0/A:col10/1731259484044/Put/seqid=0 2024-11-10T17:24:45,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742366_1542 (size=30955) 2024-11-10T17:24:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T17:24:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259546205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259546206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259546206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259546207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259546212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,214 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,222 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/315bed0e0d7748098270fe997b92211c 2024-11-10T17:24:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/79ab6c53870a4dd0a96457a7418be859 is 50, key is test_row_0/B:col10/1731259484044/Put/seqid=0 2024-11-10T17:24:46,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742367_1543 (size=12001) 2024-11-10T17:24:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259546312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259546312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259546312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259546321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259546522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259546522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259546522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259546531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,633 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/79ab6c53870a4dd0a96457a7418be859 2024-11-10T17:24:46,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/13635f524b4c4094b6bcc8e196f82e1d is 50, key is test_row_0/C:col10/1731259484044/Put/seqid=0 2024-11-10T17:24:46,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742368_1544 (size=12001) 2024-11-10T17:24:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259546831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259546831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259546831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:46,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259546842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/13635f524b4c4094b6bcc8e196f82e1d 2024-11-10T17:24:47,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/315bed0e0d7748098270fe997b92211c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/315bed0e0d7748098270fe997b92211c 2024-11-10T17:24:47,057 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/315bed0e0d7748098270fe997b92211c, entries=150, sequenceid=41, filesize=30.2 K 2024-11-10T17:24:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/79ab6c53870a4dd0a96457a7418be859 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/79ab6c53870a4dd0a96457a7418be859 2024-11-10T17:24:47,061 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/79ab6c53870a4dd0a96457a7418be859, entries=150, sequenceid=41, filesize=11.7 K 2024-11-10T17:24:47,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/13635f524b4c4094b6bcc8e196f82e1d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/13635f524b4c4094b6bcc8e196f82e1d 2024-11-10T17:24:47,064 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/13635f524b4c4094b6bcc8e196f82e1d, entries=150, sequenceid=41, filesize=11.7 K 2024-11-10T17:24:47,065 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for da5a26d0da1375a506f6b0241d015eea in 1677ms, sequenceid=41, compaction requested=false 2024-11-10T17:24:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-10T17:24:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-10T17:24:47,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-10T17:24:47,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0550 sec 2024-11-10T17:24:47,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 3.0600 sec 2024-11-10T17:24:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:47,344 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:47,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:24:47,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110556f5c7c12e24f5eb6acbaa4b859047b_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259487344/Put/seqid=0 2024-11-10T17:24:47,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742369_1545 (size=17034) 2024-11-10T17:24:47,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259547371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259547372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259547373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259547374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259547482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259547482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259547482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259547482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259547692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259547692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259547692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259547692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:47,756 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:47,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110556f5c7c12e24f5eb6acbaa4b859047b_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110556f5c7c12e24f5eb6acbaa4b859047b_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:47,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/5ef026276dd44db3b52706990bd124c8, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:47,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/5ef026276dd44db3b52706990bd124c8 is 175, key is test_row_0/A:col10/1731259487344/Put/seqid=0 2024-11-10T17:24:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742370_1546 (size=48139) 2024-11-10T17:24:48,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259548001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259548001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259548001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259548002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/5ef026276dd44db3b52706990bd124c8 2024-11-10T17:24:48,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/82dd721f04b94c0886ad4b73cf424aa8 is 50, key is test_row_0/B:col10/1731259487344/Put/seqid=0 2024-11-10T17:24:48,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742371_1547 (size=12001) 2024-11-10T17:24:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T17:24:48,181 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:48,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-10T17:24:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T17:24:48,183 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:48,184 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:48,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:48,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259548231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,235 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_0' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T17:24:48,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-10T17:24:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:48,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:48,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:48,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-10T17:24:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T17:24:48,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259548512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259548512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259548512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259548512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:48,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:48,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:48,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/82dd721f04b94c0886ad4b73cf424aa8 2024-11-10T17:24:48,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/58cb5f1e9d9c46bea3667a92719e362c is 50, key is test_row_0/C:col10/1731259487344/Put/seqid=0 2024-11-10T17:24:48,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742372_1548 (size=12001) 2024-11-10T17:24:48,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/58cb5f1e9d9c46bea3667a92719e362c 2024-11-10T17:24:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/5ef026276dd44db3b52706990bd124c8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/5ef026276dd44db3b52706990bd124c8 2024-11-10T17:24:48,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/5ef026276dd44db3b52706990bd124c8, entries=250, sequenceid=55, filesize=47.0 K 2024-11-10T17:24:48,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/82dd721f04b94c0886ad4b73cf424aa8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/82dd721f04b94c0886ad4b73cf424aa8 2024-11-10T17:24:48,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/82dd721f04b94c0886ad4b73cf424aa8, entries=150, sequenceid=55, filesize=11.7 K 2024-11-10T17:24:48,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/58cb5f1e9d9c46bea3667a92719e362c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/58cb5f1e9d9c46bea3667a92719e362c 2024-11-10T17:24:48,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/58cb5f1e9d9c46bea3667a92719e362c, entries=150, sequenceid=55, filesize=11.7 K 2024-11-10T17:24:48,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for da5a26d0da1375a506f6b0241d015eea in 1256ms, sequenceid=55, compaction requested=true 2024-11-10T17:24:48,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:48,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:48,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:48,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:48,601 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:48,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:48,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:48,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:48,601 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:48,603 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:48,603 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:48,603 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/B is initiating minor compaction (all files) 2024-11-10T17:24:48,603 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/A is initiating minor compaction (all files) 2024-11-10T17:24:48,603 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/A in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,603 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/B in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,603 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c5e7cb6c106f4d41bb8667a994fc87f6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/315bed0e0d7748098270fe997b92211c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/5ef026276dd44db3b52706990bd124c8] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=124.3 K 2024-11-10T17:24:48,603 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/dd4c62422bfa4765a7abb88d79dafca1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/79ab6c53870a4dd0a96457a7418be859, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/82dd721f04b94c0886ad4b73cf424aa8] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=35.2 K 2024-11-10T17:24:48,603 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,603 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c5e7cb6c106f4d41bb8667a994fc87f6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/315bed0e0d7748098270fe997b92211c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/5ef026276dd44db3b52706990bd124c8] 2024-11-10T17:24:48,604 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting dd4c62422bfa4765a7abb88d79dafca1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731259484013 2024-11-10T17:24:48,604 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e7cb6c106f4d41bb8667a994fc87f6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731259484013 2024-11-10T17:24:48,604 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 79ab6c53870a4dd0a96457a7418be859, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731259484039 2024-11-10T17:24:48,604 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 315bed0e0d7748098270fe997b92211c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731259484039 2024-11-10T17:24:48,604 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 82dd721f04b94c0886ad4b73cf424aa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259486204 2024-11-10T17:24:48,605 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef026276dd44db3b52706990bd124c8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259486204 2024-11-10T17:24:48,610 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:48,612 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100ea2db07744743babacc0641870905ef_da5a26d0da1375a506f6b0241d015eea store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:48,613 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:48,613 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/6392cf94344944d9a70523ae67d9aaa3 is 50, key is test_row_0/B:col10/1731259487344/Put/seqid=0 2024-11-10T17:24:48,614 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100ea2db07744743babacc0641870905ef_da5a26d0da1375a506f6b0241d015eea, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:48,614 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100ea2db07744743babacc0641870905ef_da5a26d0da1375a506f6b0241d015eea because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742374_1550 (size=4469) 2024-11-10T17:24:48,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742373_1549 (size=12104) 2024-11-10T17:24:48,629 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/6392cf94344944d9a70523ae67d9aaa3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/6392cf94344944d9a70523ae67d9aaa3 2024-11-10T17:24:48,634 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/B of da5a26d0da1375a506f6b0241d015eea into 6392cf94344944d9a70523ae67d9aaa3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:48,634 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:48,634 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/B, priority=13, startTime=1731259488600; duration=0sec 2024-11-10T17:24:48,634 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:48,634 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:B 2024-11-10T17:24:48,634 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:48,635 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:48,635 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/C is initiating minor compaction (all files) 2024-11-10T17:24:48,635 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/C in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,635 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b3c29464ff584fbe8afdd7ab8c1a0447, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/13635f524b4c4094b6bcc8e196f82e1d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/58cb5f1e9d9c46bea3667a92719e362c] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=35.2 K 2024-11-10T17:24:48,635 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b3c29464ff584fbe8afdd7ab8c1a0447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731259484013 2024-11-10T17:24:48,636 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 13635f524b4c4094b6bcc8e196f82e1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731259484039 2024-11-10T17:24:48,636 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cb5f1e9d9c46bea3667a92719e362c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259486204 2024-11-10T17:24:48,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-10T17:24:48,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:48,642 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T17:24:48,643 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:48,643 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/6954eb7cb3ad4d01be59b0046ccdb6a1 is 50, key is test_row_0/C:col10/1731259487344/Put/seqid=0 2024-11-10T17:24:48,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e285d7e0af72475684f15d5f19126947_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259487371/Put/seqid=0 2024-11-10T17:24:48,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742375_1551 (size=12104) 2024-11-10T17:24:48,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742376_1552 (size=12154) 2024-11-10T17:24:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T17:24:49,019 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:49,019 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/771c70861f1a43e3b71b0bd488ffde1b is 175, key is test_row_0/A:col10/1731259487344/Put/seqid=0 2024-11-10T17:24:49,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742377_1553 (size=31058) 2024-11-10T17:24:49,065 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/6954eb7cb3ad4d01be59b0046ccdb6a1 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/6954eb7cb3ad4d01be59b0046ccdb6a1 2024-11-10T17:24:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:49,069 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/C of da5a26d0da1375a506f6b0241d015eea into 6954eb7cb3ad4d01be59b0046ccdb6a1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:49,069 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:49,069 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/C, priority=13, startTime=1731259488601; duration=0sec 2024-11-10T17:24:49,069 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:49,070 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:C 2024-11-10T17:24:49,070 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e285d7e0af72475684f15d5f19126947_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e285d7e0af72475684f15d5f19126947_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:49,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a0ef63225e1f4ac1ba0424c66cdfb417, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:49,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a0ef63225e1f4ac1ba0424c66cdfb417 is 175, key is test_row_0/A:col10/1731259487371/Put/seqid=0 2024-11-10T17:24:49,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742378_1554 (size=30955) 2024-11-10T17:24:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T17:24:49,428 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/771c70861f1a43e3b71b0bd488ffde1b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/771c70861f1a43e3b71b0bd488ffde1b 2024-11-10T17:24:49,432 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/A of da5a26d0da1375a506f6b0241d015eea into 771c70861f1a43e3b71b0bd488ffde1b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:49,432 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:49,432 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/A, priority=13, startTime=1731259488600; duration=0sec 2024-11-10T17:24:49,432 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:49,432 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:A 2024-11-10T17:24:49,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a0ef63225e1f4ac1ba0424c66cdfb417 2024-11-10T17:24:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e9fe16c4ebc8477b90d5fa44a65737f8 is 50, key is test_row_0/B:col10/1731259487371/Put/seqid=0 2024-11-10T17:24:49,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742379_1555 (size=12001) 2024-11-10T17:24:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:49,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259549537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259549538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259549539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259549540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259549651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259549651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259549651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259549652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259549861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259549861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259549862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259549862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:49,893 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e9fe16c4ebc8477b90d5fa44a65737f8 2024-11-10T17:24:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/d0096e34321d44078537c041b0231e1b is 50, key is test_row_0/C:col10/1731259487371/Put/seqid=0 2024-11-10T17:24:49,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742380_1556 (size=12001) 2024-11-10T17:24:50,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259550172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259550172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259550172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259550172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,303 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/d0096e34321d44078537c041b0231e1b 2024-11-10T17:24:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a0ef63225e1f4ac1ba0424c66cdfb417 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a0ef63225e1f4ac1ba0424c66cdfb417 2024-11-10T17:24:50,311 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a0ef63225e1f4ac1ba0424c66cdfb417, entries=150, sequenceid=78, filesize=30.2 K 2024-11-10T17:24:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e9fe16c4ebc8477b90d5fa44a65737f8 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e9fe16c4ebc8477b90d5fa44a65737f8 2024-11-10T17:24:50,315 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e9fe16c4ebc8477b90d5fa44a65737f8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T17:24:50,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/d0096e34321d44078537c041b0231e1b as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/d0096e34321d44078537c041b0231e1b 2024-11-10T17:24:50,319 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/d0096e34321d44078537c041b0231e1b, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T17:24:50,320 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for da5a26d0da1375a506f6b0241d015eea in 1678ms, sequenceid=78, compaction requested=false 2024-11-10T17:24:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-10T17:24:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-10T17:24:50,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-10T17:24:50,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1380 sec 2024-11-10T17:24:50,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.1430 sec 2024-11-10T17:24:50,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T17:24:50,342 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:50,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-10T17:24:50,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T17:24:50,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:50,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T17:24:50,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T17:24:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:50,497 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T17:24:50,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e4ee129954f4ae391874e0ee62d29cd_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259489538/Put/seqid=0 2024-11-10T17:24:50,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742381_1557 (size=12154) 2024-11-10T17:24:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T17:24:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:50,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:50,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259550706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259550710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259550710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259550711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259550822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259550822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259550822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259550822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:50,913 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e4ee129954f4ae391874e0ee62d29cd_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e4ee129954f4ae391874e0ee62d29cd_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/cd80713c3c0e4fc6ac1f3b437819f2cb, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/cd80713c3c0e4fc6ac1f3b437819f2cb is 175, key is test_row_0/A:col10/1731259489538/Put/seqid=0 2024-11-10T17:24:50,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742382_1558 (size=30955) 2024-11-10T17:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T17:24:51,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,318 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/cd80713c3c0e4fc6ac1f3b437819f2cb 2024-11-10T17:24:51,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e63a84241d894a53be9ca2e09613dfab is 50, key is test_row_0/B:col10/1731259489538/Put/seqid=0 2024-11-10T17:24:51,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742383_1559 (size=12001) 2024-11-10T17:24:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259551341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259551341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259551342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259551342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T17:24:51,729 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e63a84241d894a53be9ca2e09613dfab 2024-11-10T17:24:51,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/85e19ee45e354340be7152e2d12e878f is 50, key is test_row_0/C:col10/1731259489538/Put/seqid=0 2024-11-10T17:24:51,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742384_1560 (size=12001) 2024-11-10T17:24:51,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259551851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/85e19ee45e354340be7152e2d12e878f 2024-11-10T17:24:52,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/cd80713c3c0e4fc6ac1f3b437819f2cb as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/cd80713c3c0e4fc6ac1f3b437819f2cb 2024-11-10T17:24:52,151 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/cd80713c3c0e4fc6ac1f3b437819f2cb, entries=150, sequenceid=94, filesize=30.2 K 2024-11-10T17:24:52,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e63a84241d894a53be9ca2e09613dfab as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e63a84241d894a53be9ca2e09613dfab 2024-11-10T17:24:52,154 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e63a84241d894a53be9ca2e09613dfab, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T17:24:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/85e19ee45e354340be7152e2d12e878f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/85e19ee45e354340be7152e2d12e878f 2024-11-10T17:24:52,158 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/85e19ee45e354340be7152e2d12e878f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T17:24:52,158 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for da5a26d0da1375a506f6b0241d015eea in 1661ms, sequenceid=94, compaction requested=true 2024-11-10T17:24:52,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:52,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:52,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-10T17:24:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-10T17:24:52,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-10T17:24:52,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-10T17:24:52,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.8200 sec 2024-11-10T17:24:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:52,275 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:52,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T17:24:52,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d60f6bd507e64ac9a8c664b958bf914c_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259492274/Put/seqid=0 2024-11-10T17:24:52,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742385_1561 (size=14594) 2024-11-10T17:24:52,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:52,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d60f6bd507e64ac9a8c664b958bf914c_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d60f6bd507e64ac9a8c664b958bf914c_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:52,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c35ad4793f0e4348a1197e759ca90815, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:52,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c35ad4793f0e4348a1197e759ca90815 is 175, key is test_row_0/A:col10/1731259492274/Put/seqid=0 2024-11-10T17:24:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742386_1562 (size=39549) 2024-11-10T17:24:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259552313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259552421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T17:24:52,492 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:52,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-10T17:24:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T17:24:52,494 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:52,495 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:52,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T17:24:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259552631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T17:24:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:52,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c35ad4793f0e4348a1197e759ca90815 2024-11-10T17:24:52,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/561b166b742b4fb795748b8f33d59791 is 50, key is test_row_0/B:col10/1731259492274/Put/seqid=0 2024-11-10T17:24:52,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742387_1563 (size=12001) 2024-11-10T17:24:52,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T17:24:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:52,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T17:24:52,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259552862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259552862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,865 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259552862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,865 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259552862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,866 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,866 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259552943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:52,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T17:24:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:52,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T17:24:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/561b166b742b4fb795748b8f33d59791 2024-11-10T17:24:53,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/c3072f94557446cba73c893a5f15485c is 50, key is test_row_0/C:col10/1731259492274/Put/seqid=0 2024-11-10T17:24:53,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742388_1564 (size=12001) 2024-11-10T17:24:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T17:24:53,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T17:24:53,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:53,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T17:24:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:53,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259553461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:53,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/c3072f94557446cba73c893a5f15485c 2024-11-10T17:24:53,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/c35ad4793f0e4348a1197e759ca90815 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c35ad4793f0e4348a1197e759ca90815 2024-11-10T17:24:53,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c35ad4793f0e4348a1197e759ca90815, entries=200, sequenceid=119, filesize=38.6 K 2024-11-10T17:24:53,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/561b166b742b4fb795748b8f33d59791 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/561b166b742b4fb795748b8f33d59791 2024-11-10T17:24:53,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/561b166b742b4fb795748b8f33d59791, entries=150, sequenceid=119, filesize=11.7 K 2024-11-10T17:24:53,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/c3072f94557446cba73c893a5f15485c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/c3072f94557446cba73c893a5f15485c 2024-11-10T17:24:53,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/c3072f94557446cba73c893a5f15485c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-10T17:24:53,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for da5a26d0da1375a506f6b0241d015eea in 1260ms, sequenceid=119, compaction requested=true 2024-11-10T17:24:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:53,535 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:24:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:53,535 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:24:53,536 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:24:53,536 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:24:53,536 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/A is initiating minor compaction (all files) 2024-11-10T17:24:53,536 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/B is initiating minor compaction (all files) 2024-11-10T17:24:53,537 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/A in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,537 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/B in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,537 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/771c70861f1a43e3b71b0bd488ffde1b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a0ef63225e1f4ac1ba0424c66cdfb417, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/cd80713c3c0e4fc6ac1f3b437819f2cb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c35ad4793f0e4348a1197e759ca90815] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=129.4 K 2024-11-10T17:24:53,537 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/6392cf94344944d9a70523ae67d9aaa3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e9fe16c4ebc8477b90d5fa44a65737f8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e63a84241d894a53be9ca2e09613dfab, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/561b166b742b4fb795748b8f33d59791] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=47.0 K 2024-11-10T17:24:53,537 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,537 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/771c70861f1a43e3b71b0bd488ffde1b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a0ef63225e1f4ac1ba0424c66cdfb417, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/cd80713c3c0e4fc6ac1f3b437819f2cb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c35ad4793f0e4348a1197e759ca90815] 2024-11-10T17:24:53,537 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6392cf94344944d9a70523ae67d9aaa3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259486204 2024-11-10T17:24:53,537 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771c70861f1a43e3b71b0bd488ffde1b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259486204 2024-11-10T17:24:53,537 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ef63225e1f4ac1ba0424c66cdfb417, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731259487365 2024-11-10T17:24:53,537 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e9fe16c4ebc8477b90d5fa44a65737f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731259487365 2024-11-10T17:24:53,538 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e63a84241d894a53be9ca2e09613dfab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259489535 2024-11-10T17:24:53,538 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd80713c3c0e4fc6ac1f3b437819f2cb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259489535 2024-11-10T17:24:53,538 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 561b166b742b4fb795748b8f33d59791, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731259490697 2024-11-10T17:24:53,538 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35ad4793f0e4348a1197e759ca90815, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731259490697 2024-11-10T17:24:53,545 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:53,546 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:53,547 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102918da0345594042ade9f3d17be407fc_da5a26d0da1375a506f6b0241d015eea store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:53,547 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/c91fe3cf7dad4f50bfe75e2a39fb5c33 is 50, key is test_row_0/B:col10/1731259492274/Put/seqid=0 2024-11-10T17:24:53,549 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102918da0345594042ade9f3d17be407fc_da5a26d0da1375a506f6b0241d015eea, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:53,549 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102918da0345594042ade9f3d17be407fc_da5a26d0da1375a506f6b0241d015eea because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:53,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742389_1565 (size=12241) 2024-11-10T17:24:53,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742390_1566 (size=4469) 2024-11-10T17:24:53,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T17:24:53,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,563 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T17:24:53,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b1e403624a89477a86195dd109c309a1_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259492307/Put/seqid=0 2024-11-10T17:24:53,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742391_1567 (size=12204) 2024-11-10T17:24:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T17:24:53,954 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:53,954 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/4d4b4d896cd441d19cf81d70ac37942d is 175, key is test_row_0/A:col10/1731259492274/Put/seqid=0 2024-11-10T17:24:53,957 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/c91fe3cf7dad4f50bfe75e2a39fb5c33 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/c91fe3cf7dad4f50bfe75e2a39fb5c33 2024-11-10T17:24:53,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742392_1568 (size=31195) 2024-11-10T17:24:53,962 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/B of da5a26d0da1375a506f6b0241d015eea into c91fe3cf7dad4f50bfe75e2a39fb5c33(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:53,962 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:53,962 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/B, priority=12, startTime=1731259493535; duration=0sec 2024-11-10T17:24:53,962 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:53,962 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:B 2024-11-10T17:24:53,962 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T17:24:53,964 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/4d4b4d896cd441d19cf81d70ac37942d as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/4d4b4d896cd441d19cf81d70ac37942d 2024-11-10T17:24:53,964 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T17:24:53,964 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/C is initiating minor compaction (all files) 2024-11-10T17:24:53,964 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/C in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:53,964 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/6954eb7cb3ad4d01be59b0046ccdb6a1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/d0096e34321d44078537c041b0231e1b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/85e19ee45e354340be7152e2d12e878f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/c3072f94557446cba73c893a5f15485c] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=47.0 K 2024-11-10T17:24:53,964 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 6954eb7cb3ad4d01be59b0046ccdb6a1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731259486204 2024-11-10T17:24:53,965 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting d0096e34321d44078537c041b0231e1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731259487365 2024-11-10T17:24:53,965 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e19ee45e354340be7152e2d12e878f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731259489535 2024-11-10T17:24:53,965 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c3072f94557446cba73c893a5f15485c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731259490697 2024-11-10T17:24:53,969 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/A of da5a26d0da1375a506f6b0241d015eea into 4d4b4d896cd441d19cf81d70ac37942d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:53,969 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:53,969 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/A, priority=12, startTime=1731259493535; duration=0sec 2024-11-10T17:24:53,969 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:53,969 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:A 2024-11-10T17:24:53,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:53,974 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:53,974 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/cecb1a6ba50448cbbef49a638b8b38f6 is 50, key is test_row_0/C:col10/1731259492274/Put/seqid=0 2024-11-10T17:24:53,976 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b1e403624a89477a86195dd109c309a1_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1e403624a89477a86195dd109c309a1_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/8d6b927c4a5a40269c42158dce05cb7c, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:53,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/8d6b927c4a5a40269c42158dce05cb7c is 175, key is test_row_0/A:col10/1731259492307/Put/seqid=0 2024-11-10T17:24:53,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742394_1570 (size=31005) 2024-11-10T17:24:53,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742393_1569 (size=12241) 2024-11-10T17:24:54,381 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/8d6b927c4a5a40269c42158dce05cb7c 2024-11-10T17:24:54,385 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/cecb1a6ba50448cbbef49a638b8b38f6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/cecb1a6ba50448cbbef49a638b8b38f6 2024-11-10T17:24:54,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/74b83399b0ec494bbddc90ebaac75116 is 50, key is test_row_0/B:col10/1731259492307/Put/seqid=0 2024-11-10T17:24:54,390 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/C of da5a26d0da1375a506f6b0241d015eea into cecb1a6ba50448cbbef49a638b8b38f6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:54,390 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:54,390 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/C, priority=12, startTime=1731259493535; duration=0sec 2024-11-10T17:24:54,390 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:54,390 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:C 2024-11-10T17:24:54,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742395_1571 (size=12051) 2024-11-10T17:24:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:54,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:54,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259554560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T17:24:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259554671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:54,793 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/74b83399b0ec494bbddc90ebaac75116 2024-11-10T17:24:54,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/f38b77d471714ea09e502258025b49b2 is 50, key is test_row_0/C:col10/1731259492307/Put/seqid=0 2024-11-10T17:24:54,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742396_1572 (size=12051) 2024-11-10T17:24:54,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259554881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259554881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259554881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:54,884 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,884 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_2' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259554881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:54,884 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259554891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:54,893 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:24:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:55,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259555192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:55,203 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/f38b77d471714ea09e502258025b49b2 2024-11-10T17:24:55,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/8d6b927c4a5a40269c42158dce05cb7c as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/8d6b927c4a5a40269c42158dce05cb7c 2024-11-10T17:24:55,210 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/8d6b927c4a5a40269c42158dce05cb7c, entries=150, sequenceid=130, filesize=30.3 K 2024-11-10T17:24:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/74b83399b0ec494bbddc90ebaac75116 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/74b83399b0ec494bbddc90ebaac75116 2024-11-10T17:24:55,214 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/74b83399b0ec494bbddc90ebaac75116, entries=150, sequenceid=130, filesize=11.8 K 2024-11-10T17:24:55,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/f38b77d471714ea09e502258025b49b2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/f38b77d471714ea09e502258025b49b2 2024-11-10T17:24:55,217 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/f38b77d471714ea09e502258025b49b2, entries=150, sequenceid=130, filesize=11.8 K 2024-11-10T17:24:55,218 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for da5a26d0da1375a506f6b0241d015eea in 1655ms, sequenceid=130, compaction requested=false 2024-11-10T17:24:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-10T17:24:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-10T17:24:55,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-10T17:24:55,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7250 sec 2024-11-10T17:24:55,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.7290 sec 2024-11-10T17:24:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:55,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T17:24:55,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a94f51f02e54e7cba756e9f98210948_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259494553/Put/seqid=0 2024-11-10T17:24:55,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742397_1573 (size=14794) 2024-11-10T17:24:55,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259555732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:55,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259555841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259556051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:56,116 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:56,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a94f51f02e54e7cba756e9f98210948_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a94f51f02e54e7cba756e9f98210948_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:56,119 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/48ca8c75bf1a415faf4312cbb64a040a, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:56,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/48ca8c75bf1a415faf4312cbb64a040a is 175, key is test_row_0/A:col10/1731259494553/Put/seqid=0 2024-11-10T17:24:56,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742398_1574 (size=39749) 2024-11-10T17:24:56,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259556361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:56,524 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/48ca8c75bf1a415faf4312cbb64a040a 2024-11-10T17:24:56,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/5547a90e77a34d7e836499c4ad79bf3e is 50, key is test_row_0/B:col10/1731259494553/Put/seqid=0 2024-11-10T17:24:56,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742399_1575 (size=12151) 2024-11-10T17:24:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T17:24:56,652 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:56,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-10T17:24:56,654 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T17:24:56,655 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:56,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T17:24:56,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T17:24:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:56,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259556871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:56,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/5547a90e77a34d7e836499c4ad79bf3e 2024-11-10T17:24:56,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/b7b8fa95064942a7a29ded32a35ee9df is 50, key is test_row_0/C:col10/1731259494553/Put/seqid=0 2024-11-10T17:24:56,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742400_1576 (size=12151) 2024-11-10T17:24:56,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T17:24:56,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:56,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:56,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T17:24:57,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T17:24:57,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:57,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:57,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:57,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T17:24:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T17:24:57,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/b7b8fa95064942a7a29ded32a35ee9df 2024-11-10T17:24:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/48ca8c75bf1a415faf4312cbb64a040a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/48ca8c75bf1a415faf4312cbb64a040a 2024-11-10T17:24:57,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/48ca8c75bf1a415faf4312cbb64a040a, entries=200, sequenceid=159, filesize=38.8 K 2024-11-10T17:24:57,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/5547a90e77a34d7e836499c4ad79bf3e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/5547a90e77a34d7e836499c4ad79bf3e 2024-11-10T17:24:57,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/5547a90e77a34d7e836499c4ad79bf3e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T17:24:57,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/b7b8fa95064942a7a29ded32a35ee9df as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b7b8fa95064942a7a29ded32a35ee9df 2024-11-10T17:24:57,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b7b8fa95064942a7a29ded32a35ee9df, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T17:24:57,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for da5a26d0da1375a506f6b0241d015eea in 1656ms, sequenceid=159, compaction requested=true 2024-11-10T17:24:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:57,360 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:57,360 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:57,361 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:57,361 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:57,361 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/B is initiating minor compaction (all files) 2024-11-10T17:24:57,361 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/A is initiating minor compaction (all files) 2024-11-10T17:24:57,361 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/B in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,361 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/A in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,362 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/c91fe3cf7dad4f50bfe75e2a39fb5c33, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/74b83399b0ec494bbddc90ebaac75116, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/5547a90e77a34d7e836499c4ad79bf3e] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=35.6 K 2024-11-10T17:24:57,362 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/4d4b4d896cd441d19cf81d70ac37942d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/8d6b927c4a5a40269c42158dce05cb7c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/48ca8c75bf1a415faf4312cbb64a040a] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=99.6 K 2024-11-10T17:24:57,362 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,362 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/4d4b4d896cd441d19cf81d70ac37942d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/8d6b927c4a5a40269c42158dce05cb7c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/48ca8c75bf1a415faf4312cbb64a040a] 2024-11-10T17:24:57,362 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting c91fe3cf7dad4f50bfe75e2a39fb5c33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731259490697 2024-11-10T17:24:57,362 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d4b4d896cd441d19cf81d70ac37942d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731259490697 2024-11-10T17:24:57,362 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b83399b0ec494bbddc90ebaac75116, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731259492295 2024-11-10T17:24:57,362 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d6b927c4a5a40269c42158dce05cb7c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731259492295 2024-11-10T17:24:57,362 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 5547a90e77a34d7e836499c4ad79bf3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731259494553 2024-11-10T17:24:57,362 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48ca8c75bf1a415faf4312cbb64a040a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731259494530 2024-11-10T17:24:57,367 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:57,369 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111051f50ca30cd0403ebe857ab7e7628a57_da5a26d0da1375a506f6b0241d015eea store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:57,370 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#B#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:57,371 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111051f50ca30cd0403ebe857ab7e7628a57_da5a26d0da1375a506f6b0241d015eea, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:57,371 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/1bc875bcc517411e901dd1770169c6e2 is 50, key is test_row_0/B:col10/1731259494553/Put/seqid=0 2024-11-10T17:24:57,371 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111051f50ca30cd0403ebe857ab7e7628a57_da5a26d0da1375a506f6b0241d015eea because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:57,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742401_1577 (size=12493) 2024-11-10T17:24:57,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742402_1578 (size=4469) 2024-11-10T17:24:57,389 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#A#compaction#486 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:57,390 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/bd71cabb3b0a48f5b2b0023e0367e3b6 is 175, key is test_row_0/A:col10/1731259494553/Put/seqid=0 2024-11-10T17:24:57,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742403_1579 (size=31447) 2024-11-10T17:24:57,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T17:24:57,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,420 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T17:24:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111040d50bd79a3448a38bb9e407036e02f9_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259495730/Put/seqid=0 2024-11-10T17:24:57,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742404_1580 (size=12304) 2024-11-10T17:24:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:57,438 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111040d50bd79a3448a38bb9e407036e02f9_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111040d50bd79a3448a38bb9e407036e02f9_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:57,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/10c750778ec84bbf9ed012523f6563bc, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:57,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/10c750778ec84bbf9ed012523f6563bc is 175, key is test_row_0/A:col10/1731259495730/Put/seqid=0 2024-11-10T17:24:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742405_1581 (size=31105) 2024-11-10T17:24:57,782 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/1bc875bcc517411e901dd1770169c6e2 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/1bc875bcc517411e901dd1770169c6e2 2024-11-10T17:24:57,786 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/B of da5a26d0da1375a506f6b0241d015eea into 1bc875bcc517411e901dd1770169c6e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:57,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:57,787 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/B, priority=13, startTime=1731259497360; duration=0sec 2024-11-10T17:24:57,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:57,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:B 2024-11-10T17:24:57,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:57,787 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:57,788 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/C is initiating minor compaction (all files) 2024-11-10T17:24:57,788 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/C in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:57,788 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/cecb1a6ba50448cbbef49a638b8b38f6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/f38b77d471714ea09e502258025b49b2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b7b8fa95064942a7a29ded32a35ee9df] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=35.6 K 2024-11-10T17:24:57,788 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting cecb1a6ba50448cbbef49a638b8b38f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731259490697 2024-11-10T17:24:57,788 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f38b77d471714ea09e502258025b49b2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731259492295 2024-11-10T17:24:57,788 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b8fa95064942a7a29ded32a35ee9df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731259494553 2024-11-10T17:24:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T17:24:57,795 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:57,795 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/7106c049d95a4bc1a8b19b282656fd92 is 50, key is test_row_0/C:col10/1731259494553/Put/seqid=0 2024-11-10T17:24:57,797 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/bd71cabb3b0a48f5b2b0023e0367e3b6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bd71cabb3b0a48f5b2b0023e0367e3b6 2024-11-10T17:24:57,801 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/A of da5a26d0da1375a506f6b0241d015eea into bd71cabb3b0a48f5b2b0023e0367e3b6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:57,801 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:57,801 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/A, priority=13, startTime=1731259497360; duration=0sec 2024-11-10T17:24:57,801 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:57,802 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:A 2024-11-10T17:24:57,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742406_1582 (size=12493) 2024-11-10T17:24:57,843 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/10c750778ec84bbf9ed012523f6563bc 2024-11-10T17:24:57,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/745c0839fb7a4c82973ee063e588cd05 is 50, key is test_row_0/B:col10/1731259495730/Put/seqid=0 2024-11-10T17:24:57,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742407_1583 (size=12151) 2024-11-10T17:24:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:57,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:57,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259557973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:58,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259558081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,218 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/7106c049d95a4bc1a8b19b282656fd92 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7106c049d95a4bc1a8b19b282656fd92 2024-11-10T17:24:58,222 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/C of da5a26d0da1375a506f6b0241d015eea into 7106c049d95a4bc1a8b19b282656fd92(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:58,222 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:58,222 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/C, priority=13, startTime=1731259497360; duration=0sec 2024-11-10T17:24:58,222 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:58,222 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:C 2024-11-10T17:24:58,270 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/745c0839fb7a4c82973ee063e588cd05 2024-11-10T17:24:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/a2a0f7ba7ea94c2da220075cc8d7ce09 is 50, key is test_row_0/C:col10/1731259495730/Put/seqid=0 2024-11-10T17:24:58,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742408_1584 (size=12151) 2024-11-10T17:24:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259558291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259558601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,686 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/a2a0f7ba7ea94c2da220075cc8d7ce09 2024-11-10T17:24:58,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/10c750778ec84bbf9ed012523f6563bc as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/10c750778ec84bbf9ed012523f6563bc 2024-11-10T17:24:58,693 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/10c750778ec84bbf9ed012523f6563bc, entries=150, sequenceid=169, filesize=30.4 K 2024-11-10T17:24:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/745c0839fb7a4c82973ee063e588cd05 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/745c0839fb7a4c82973ee063e588cd05 2024-11-10T17:24:58,697 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/745c0839fb7a4c82973ee063e588cd05, entries=150, sequenceid=169, filesize=11.9 K 2024-11-10T17:24:58,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/a2a0f7ba7ea94c2da220075cc8d7ce09 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/a2a0f7ba7ea94c2da220075cc8d7ce09 2024-11-10T17:24:58,700 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/a2a0f7ba7ea94c2da220075cc8d7ce09, entries=150, sequenceid=169, filesize=11.9 K 2024-11-10T17:24:58,701 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for da5a26d0da1375a506f6b0241d015eea in 1282ms, sequenceid=169, compaction requested=false 2024-11-10T17:24:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-10T17:24:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-10T17:24:58,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-10T17:24:58,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0470 sec 2024-11-10T17:24:58,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.0530 sec 2024-11-10T17:24:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T17:24:58,801 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:24:58,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:24:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-10T17:24:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T17:24:58,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:24:58,804 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:24:58,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:24:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:58,905 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:24:58,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T17:24:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T17:24:58,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107aca3a51d55b4698b4afc579aaf3628c_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259497961/Put/seqid=0 2024-11-10T17:24:58,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742409_1585 (size=14794) 2024-11-10T17:24:58,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:58,923 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107aca3a51d55b4698b4afc579aaf3628c_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107aca3a51d55b4698b4afc579aaf3628c_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:58,923 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/bfbc7e527870459e9e3e1ec7595eaf8a, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:58,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/bfbc7e527870459e9e3e1ec7595eaf8a is 175, key is test_row_0/A:col10/1731259497961/Put/seqid=0 2024-11-10T17:24:58,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742410_1586 (size=39749) 2024-11-10T17:24:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259558924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259558923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259558925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259558925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:58,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T17:24:58,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:58,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:58,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:58,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:58,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259559031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259559031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259559032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259559042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T17:24:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259559112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T17:24:59,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259559242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259559242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259559242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259559251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T17:24:59,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:59,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,329 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/bfbc7e527870459e9e3e1ec7595eaf8a 2024-11-10T17:24:59,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/849bb094cbae42199ce8c3e54e4b237a is 50, key is test_row_0/B:col10/1731259497961/Put/seqid=0 2024-11-10T17:24:59,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742411_1587 (size=12151) 2024-11-10T17:24:59,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/849bb094cbae42199ce8c3e54e4b237a 2024-11-10T17:24:59,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/2812d4c48a2b4367bb0a9d6778f6984e is 50, key is test_row_0/C:col10/1731259497961/Put/seqid=0 2024-11-10T17:24:59,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742412_1588 (size=12151) 2024-11-10T17:24:59,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T17:24:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T17:24:59,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259559552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259559552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259559552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:24:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259559561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:24:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:24:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:24:59,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T17:24:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T17:24:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:24:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:24:59,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/2812d4c48a2b4367bb0a9d6778f6984e 2024-11-10T17:24:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/bfbc7e527870459e9e3e1ec7595eaf8a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bfbc7e527870459e9e3e1ec7595eaf8a 2024-11-10T17:24:59,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bfbc7e527870459e9e3e1ec7595eaf8a, entries=200, sequenceid=199, filesize=38.8 K 2024-11-10T17:24:59,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/849bb094cbae42199ce8c3e54e4b237a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/849bb094cbae42199ce8c3e54e4b237a 2024-11-10T17:24:59,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/849bb094cbae42199ce8c3e54e4b237a, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T17:24:59,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/2812d4c48a2b4367bb0a9d6778f6984e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/2812d4c48a2b4367bb0a9d6778f6984e 2024-11-10T17:24:59,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/2812d4c48a2b4367bb0a9d6778f6984e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T17:24:59,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for da5a26d0da1375a506f6b0241d015eea in 901ms, sequenceid=199, compaction requested=true 2024-11-10T17:24:59,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:59,806 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:24:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:59,807 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:24:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:59,807 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:24:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:59,807 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/A is initiating minor compaction (all files) 2024-11-10T17:24:59,807 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/A in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,808 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bd71cabb3b0a48f5b2b0023e0367e3b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/10c750778ec84bbf9ed012523f6563bc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bfbc7e527870459e9e3e1ec7595eaf8a] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=99.9 K 2024-11-10T17:24:59,808 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,808 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bd71cabb3b0a48f5b2b0023e0367e3b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/10c750778ec84bbf9ed012523f6563bc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bfbc7e527870459e9e3e1ec7595eaf8a] 2024-11-10T17:24:59,810 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd71cabb3b0a48f5b2b0023e0367e3b6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731259494553 2024-11-10T17:24:59,810 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:59,810 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/B is initiating minor compaction (all files) 2024-11-10T17:24:59,810 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/B in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,810 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/1bc875bcc517411e901dd1770169c6e2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/745c0839fb7a4c82973ee063e588cd05, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/849bb094cbae42199ce8c3e54e4b237a] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=35.9 K 2024-11-10T17:24:59,810 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c750778ec84bbf9ed012523f6563bc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731259495719 2024-11-10T17:24:59,811 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc875bcc517411e901dd1770169c6e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731259494553 2024-11-10T17:24:59,811 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 745c0839fb7a4c82973ee063e588cd05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731259495719 2024-11-10T17:24:59,811 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfbc7e527870459e9e3e1ec7595eaf8a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731259497947 2024-11-10T17:24:59,813 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 849bb094cbae42199ce8c3e54e4b237a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731259497961 2024-11-10T17:24:59,823 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:59,823 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/eede62149b0c432a8fe3065b8482c337 is 50, key is test_row_0/B:col10/1731259497961/Put/seqid=0 2024-11-10T17:24:59,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742413_1589 (size=12595) 2024-11-10T17:24:59,837 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/eede62149b0c432a8fe3065b8482c337 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/eede62149b0c432a8fe3065b8482c337 2024-11-10T17:24:59,842 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:59,844 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/B of da5a26d0da1375a506f6b0241d015eea into eede62149b0c432a8fe3065b8482c337(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:59,844 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:59,844 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/B, priority=13, startTime=1731259499807; duration=0sec 2024-11-10T17:24:59,844 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:24:59,844 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:B 2024-11-10T17:24:59,844 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:24:59,846 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102adfcdbb872e40038f74fbc4ff7349a7_da5a26d0da1375a506f6b0241d015eea store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:59,846 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:24:59,846 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/C is initiating minor compaction (all files) 2024-11-10T17:24:59,847 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/C in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,847 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7106c049d95a4bc1a8b19b282656fd92, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/a2a0f7ba7ea94c2da220075cc8d7ce09, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/2812d4c48a2b4367bb0a9d6778f6984e] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=35.9 K 2024-11-10T17:24:59,847 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7106c049d95a4bc1a8b19b282656fd92, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731259494553 2024-11-10T17:24:59,847 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting a2a0f7ba7ea94c2da220075cc8d7ce09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731259495719 2024-11-10T17:24:59,848 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 2812d4c48a2b4367bb0a9d6778f6984e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731259497961 2024-11-10T17:24:59,848 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102adfcdbb872e40038f74fbc4ff7349a7_da5a26d0da1375a506f6b0241d015eea, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:59,848 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102adfcdbb872e40038f74fbc4ff7349a7_da5a26d0da1375a506f6b0241d015eea because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:59,862 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#C#compaction#497 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:59,862 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/7d0cc5a5142040b8a347e5605cba7c85 is 50, key is test_row_0/C:col10/1731259497961/Put/seqid=0 2024-11-10T17:24:59,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T17:24:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:24:59,874 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T17:24:59,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742415_1591 (size=12595) 2024-11-10T17:24:59,898 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/7d0cc5a5142040b8a347e5605cba7c85 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7d0cc5a5142040b8a347e5605cba7c85 2024-11-10T17:24:59,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742414_1590 (size=4469) 2024-11-10T17:24:59,899 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#A#compaction#496 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:24:59,900 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/75f9cb45f06b4ca9a1068d4777e4fbc5 is 175, key is test_row_0/A:col10/1731259497961/Put/seqid=0 2024-11-10T17:24:59,905 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/C of da5a26d0da1375a506f6b0241d015eea into 7d0cc5a5142040b8a347e5605cba7c85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:59,905 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:59,905 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/C, priority=13, startTime=1731259499807; duration=0sec 2024-11-10T17:24:59,905 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:59,905 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:C 2024-11-10T17:24:59,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111058f7cbf6100b47ee95e8039e98d8438d_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259498921/Put/seqid=0 2024-11-10T17:24:59,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742416_1592 (size=31549) 2024-11-10T17:24:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T17:24:59,944 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/75f9cb45f06b4ca9a1068d4777e4fbc5 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/75f9cb45f06b4ca9a1068d4777e4fbc5 2024-11-10T17:24:59,953 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/A of da5a26d0da1375a506f6b0241d015eea into 75f9cb45f06b4ca9a1068d4777e4fbc5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:24:59,953 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:24:59,953 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/A, priority=13, startTime=1731259499806; duration=0sec 2024-11-10T17:24:59,953 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:24:59,953 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:A 2024-11-10T17:24:59,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742417_1593 (size=12304) 2024-11-10T17:24:59,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:24:59,960 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111058f7cbf6100b47ee95e8039e98d8438d_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058f7cbf6100b47ee95e8039e98d8438d_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:24:59,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/013afd5e22644775b05604be748fc1b7, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:24:59,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/013afd5e22644775b05604be748fc1b7 is 175, key is test_row_0/A:col10/1731259498921/Put/seqid=0 2024-11-10T17:24:59,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742418_1594 (size=31105) 2024-11-10T17:24:59,995 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/013afd5e22644775b05604be748fc1b7 2024-11-10T17:25:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/f2472d8ce0524018a0870d8676f6cf6f is 50, key is test_row_0/B:col10/1731259498921/Put/seqid=0 2024-11-10T17:25:00,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742419_1595 (size=12151) 2024-11-10T17:25:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:00,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:25:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259560099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259560101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259560103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259560103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259560121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,125 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259560212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259560213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259560213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259560213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259560422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259560422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259560422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259560422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,452 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/f2472d8ce0524018a0870d8676f6cf6f 2024-11-10T17:25:00,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/984a9759ae8b40639d0079ae1f851f9e is 50, key is test_row_0/C:col10/1731259498921/Put/seqid=0 2024-11-10T17:25:00,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742420_1596 (size=12151) 2024-11-10T17:25:00,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259560732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259560732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259560732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259560732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:00,927 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/984a9759ae8b40639d0079ae1f851f9e 2024-11-10T17:25:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/013afd5e22644775b05604be748fc1b7 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/013afd5e22644775b05604be748fc1b7 2024-11-10T17:25:00,944 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/013afd5e22644775b05604be748fc1b7, entries=150, sequenceid=210, filesize=30.4 K 2024-11-10T17:25:00,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/f2472d8ce0524018a0870d8676f6cf6f as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2472d8ce0524018a0870d8676f6cf6f 2024-11-10T17:25:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T17:25:00,963 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2472d8ce0524018a0870d8676f6cf6f, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T17:25:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/984a9759ae8b40639d0079ae1f851f9e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/984a9759ae8b40639d0079ae1f851f9e 2024-11-10T17:25:00,970 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/984a9759ae8b40639d0079ae1f851f9e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T17:25:00,972 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for da5a26d0da1375a506f6b0241d015eea in 1098ms, sequenceid=210, compaction requested=false 2024-11-10T17:25:00,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:25:00,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-10T17:25:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-10T17:25:00,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T17:25:00,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-11-10T17:25:00,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.1790 sec 2024-11-10T17:25:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:01,244 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:25:01,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-10T17:25:01,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259561253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259561253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259561255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259561255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107b719cd053b84318a0b4d99e551afc8b_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259500101/Put/seqid=0 2024-11-10T17:25:01,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742421_1597 (size=14794) 2024-11-10T17:25:01,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:25:01,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107b719cd053b84318a0b4d99e551afc8b_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b719cd053b84318a0b4d99e551afc8b_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:01,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/36db8ea2496a4c15b3e42da334b11d19, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:01,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/36db8ea2496a4c15b3e42da334b11d19 is 175, key is test_row_0/A:col10/1731259500101/Put/seqid=0 2024-11-10T17:25:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259561361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259561361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259561374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259561374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742422_1598 (size=39749) 2024-11-10T17:25:01,396 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/36db8ea2496a4c15b3e42da334b11d19 2024-11-10T17:25:01,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e2177a514de34c1a9dfcfb696d09c036 is 50, key is test_row_0/B:col10/1731259500101/Put/seqid=0 2024-11-10T17:25:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742423_1599 (size=12151) 2024-11-10T17:25:01,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e2177a514de34c1a9dfcfb696d09c036 2024-11-10T17:25:01,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/1ae1295002764ed89b70ddbfa7356a04 is 50, key is test_row_0/C:col10/1731259500101/Put/seqid=0 2024-11-10T17:25:01,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742424_1600 (size=12151) 2024-11-10T17:25:01,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/1ae1295002764ed89b70ddbfa7356a04 2024-11-10T17:25:01,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259561571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259561571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259561582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259561582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/36db8ea2496a4c15b3e42da334b11d19 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/36db8ea2496a4c15b3e42da334b11d19 2024-11-10T17:25:01,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/36db8ea2496a4c15b3e42da334b11d19, entries=200, sequenceid=240, filesize=38.8 K 2024-11-10T17:25:01,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/e2177a514de34c1a9dfcfb696d09c036 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e2177a514de34c1a9dfcfb696d09c036 2024-11-10T17:25:01,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e2177a514de34c1a9dfcfb696d09c036, entries=150, sequenceid=240, filesize=11.9 K 2024-11-10T17:25:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/1ae1295002764ed89b70ddbfa7356a04 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/1ae1295002764ed89b70ddbfa7356a04 2024-11-10T17:25:01,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/1ae1295002764ed89b70ddbfa7356a04, entries=150, sequenceid=240, filesize=11.9 K 2024-11-10T17:25:01,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for da5a26d0da1375a506f6b0241d015eea in 371ms, sequenceid=240, compaction requested=true 2024-11-10T17:25:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:25:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T17:25:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:25:01,616 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:25:01,616 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:25:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T17:25:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:25:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da5a26d0da1375a506f6b0241d015eea:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T17:25:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:25:01,631 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:25:01,631 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:25:01,631 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/A is initiating minor compaction (all files) 2024-11-10T17:25:01,631 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/B is initiating minor compaction (all files) 2024-11-10T17:25:01,631 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/A in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:01,631 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/B in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:01,631 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/eede62149b0c432a8fe3065b8482c337, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2472d8ce0524018a0870d8676f6cf6f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e2177a514de34c1a9dfcfb696d09c036] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=36.0 K 2024-11-10T17:25:01,631 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/75f9cb45f06b4ca9a1068d4777e4fbc5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/013afd5e22644775b05604be748fc1b7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/36db8ea2496a4c15b3e42da334b11d19] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=100.0 K 2024-11-10T17:25:01,631 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:01,631 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. files: [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/75f9cb45f06b4ca9a1068d4777e4fbc5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/013afd5e22644775b05604be748fc1b7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/36db8ea2496a4c15b3e42da334b11d19] 2024-11-10T17:25:01,632 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting eede62149b0c432a8fe3065b8482c337, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731259497961 2024-11-10T17:25:01,632 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting f2472d8ce0524018a0870d8676f6cf6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731259498921 2024-11-10T17:25:01,632 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f9cb45f06b4ca9a1068d4777e4fbc5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731259497961 2024-11-10T17:25:01,635 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting e2177a514de34c1a9dfcfb696d09c036, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731259500099 2024-11-10T17:25:01,635 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013afd5e22644775b05604be748fc1b7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731259498921 2024-11-10T17:25:01,636 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36db8ea2496a4c15b3e42da334b11d19, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731259500099 2024-11-10T17:25:01,669 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:01,671 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:25:01,672 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/a97c0e0e7bf640db9fb15c0edcd9f609 is 50, key is test_row_0/B:col10/1731259500101/Put/seqid=0 2024-11-10T17:25:01,685 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111074d0133fb45048cabf2046760af32802_da5a26d0da1375a506f6b0241d015eea store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:01,687 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111074d0133fb45048cabf2046760af32802_da5a26d0da1375a506f6b0241d015eea, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:01,687 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111074d0133fb45048cabf2046760af32802_da5a26d0da1375a506f6b0241d015eea because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:01,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742425_1601 (size=12697) 2024-11-10T17:25:01,714 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/a97c0e0e7bf640db9fb15c0edcd9f609 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/a97c0e0e7bf640db9fb15c0edcd9f609 2024-11-10T17:25:01,726 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/B of da5a26d0da1375a506f6b0241d015eea into a97c0e0e7bf640db9fb15c0edcd9f609(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:25:01,726 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:25:01,726 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/B, priority=13, startTime=1731259501616; duration=0sec 2024-11-10T17:25:01,726 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T17:25:01,726 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:B 2024-11-10T17:25:01,727 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T17:25:01,728 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T17:25:01,728 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1541): da5a26d0da1375a506f6b0241d015eea/C is initiating minor compaction (all files) 2024-11-10T17:25:01,728 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da5a26d0da1375a506f6b0241d015eea/C in TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:01,728 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7d0cc5a5142040b8a347e5605cba7c85, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/984a9759ae8b40639d0079ae1f851f9e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/1ae1295002764ed89b70ddbfa7356a04] into tmpdir=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp, totalSize=36.0 K 2024-11-10T17:25:01,728 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d0cc5a5142040b8a347e5605cba7c85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731259497961 2024-11-10T17:25:01,729 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 984a9759ae8b40639d0079ae1f851f9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731259498921 2024-11-10T17:25:01,729 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae1295002764ed89b70ddbfa7356a04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731259500099 2024-11-10T17:25:01,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742426_1602 (size=4469) 2024-11-10T17:25:01,739 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#A#compaction#505 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:25:01,739 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/3fc8e205666e440c9ae5222d7130fd26 is 175, key is test_row_0/A:col10/1731259500101/Put/seqid=0 2024-11-10T17:25:01,750 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da5a26d0da1375a506f6b0241d015eea#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T17:25:01,751 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/250c6878092c43ab934cbba6c77ab942 is 50, key is test_row_0/C:col10/1731259500101/Put/seqid=0 2024-11-10T17:25:01,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742427_1603 (size=31651) 2024-11-10T17:25:01,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742428_1604 (size=12697) 2024-11-10T17:25:01,798 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/250c6878092c43ab934cbba6c77ab942 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/250c6878092c43ab934cbba6c77ab942 2024-11-10T17:25:01,808 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/C of da5a26d0da1375a506f6b0241d015eea into 250c6878092c43ab934cbba6c77ab942(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:25:01,808 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:25:01,808 INFO [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/C, priority=13, startTime=1731259501616; duration=0sec 2024-11-10T17:25:01,808 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:25:01,808 DEBUG [RS:0;8e3de9160f9f:36421-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:C 2024-11-10T17:25:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:01,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T17:25:01,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T17:25:01,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bf45ae110a0b4041b5065197f23b3ffb_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259501253/Put/seqid=0 2024-11-10T17:25:01,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742429_1605 (size=12304) 2024-11-10T17:25:01,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259561964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259561966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259561966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:01,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259561972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259562083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259562082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259562083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259562092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35832 deadline: 1731259562141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,145 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,36421,1731259326553 for 'test_row_1' in da5a26d0da1375a506f6b0241d015eea of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T17:25:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,167 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/3fc8e205666e440c9ae5222d7130fd26 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/3fc8e205666e440c9ae5222d7130fd26 2024-11-10T17:25:02,173 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da5a26d0da1375a506f6b0241d015eea/A of da5a26d0da1375a506f6b0241d015eea into 3fc8e205666e440c9ae5222d7130fd26(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T17:25:02,173 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:25:02,173 INFO [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., storeName=da5a26d0da1375a506f6b0241d015eea/A, priority=13, startTime=1731259501616; duration=0sec 2024-11-10T17:25:02,173 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T17:25:02,173 DEBUG [RS:0;8e3de9160f9f:36421-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da5a26d0da1375a506f6b0241d015eea:A 2024-11-10T17:25:02,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259562291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259562291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259562292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259562302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:25:02,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bf45ae110a0b4041b5065197f23b3ffb_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bf45ae110a0b4041b5065197f23b3ffb_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:02,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a29d330905d343eb9f7cbe6b31f27df3, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:02,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a29d330905d343eb9f7cbe6b31f27df3 is 175, key is test_row_0/A:col10/1731259501253/Put/seqid=0 2024-11-10T17:25:02,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742430_1606 (size=31105) 2024-11-10T17:25:02,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259562601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259562601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259562601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259562611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:02,755 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a29d330905d343eb9f7cbe6b31f27df3 2024-11-10T17:25:02,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/ec5f34cdd1bf45bea48ff9aada36be0a is 50, key is test_row_0/B:col10/1731259501253/Put/seqid=0 2024-11-10T17:25:02,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742431_1607 (size=12151) 2024-11-10T17:25:02,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/ec5f34cdd1bf45bea48ff9aada36be0a 2024-11-10T17:25:02,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/e2d8a3d7dfc443559d66062c32118992 is 50, key is test_row_0/C:col10/1731259501253/Put/seqid=0 2024-11-10T17:25:02,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742432_1608 (size=12151) 2024-11-10T17:25:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T17:25:02,962 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:25:02,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T17:25:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-10T17:25:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T17:25:02,966 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T17:25:02,967 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T17:25:02,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T17:25:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T17:25:03,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731259563112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:03,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35804 deadline: 1731259563112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:03,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35900 deadline: 1731259563112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:03,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T17:25:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T17:25:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:25:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:03,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:25:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:25:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35820 deadline: 1731259563121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T17:25:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 , the old value is region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T17:25:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., hostname=8e3de9160f9f,36421,1731259326553, seqNum=5 because the exception is null or not the one we care about 2024-11-10T17:25:03,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/e2d8a3d7dfc443559d66062c32118992 2024-11-10T17:25:03,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/a29d330905d343eb9f7cbe6b31f27df3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a29d330905d343eb9f7cbe6b31f27df3 2024-11-10T17:25:03,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a29d330905d343eb9f7cbe6b31f27df3, entries=150, sequenceid=254, filesize=30.4 K 2024-11-10T17:25:03,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/ec5f34cdd1bf45bea48ff9aada36be0a as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/ec5f34cdd1bf45bea48ff9aada36be0a 2024-11-10T17:25:03,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/ec5f34cdd1bf45bea48ff9aada36be0a, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T17:25:03,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/e2d8a3d7dfc443559d66062c32118992 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/e2d8a3d7dfc443559d66062c32118992 2024-11-10T17:25:03,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/e2d8a3d7dfc443559d66062c32118992, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T17:25:03,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for da5a26d0da1375a506f6b0241d015eea in 1342ms, sequenceid=254, compaction requested=false 2024-11-10T17:25:03,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:25:03,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T17:25:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:03,278 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T17:25:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T17:25:03,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a5ee9726dbaa4bcb9136c147d1d9ad42_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259501963/Put/seqid=0 2024-11-10T17:25:03,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742433_1609 (size=12454) 2024-11-10T17:25:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T17:25:03,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:25:03,733 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a5ee9726dbaa4bcb9136c147d1d9ad42_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a5ee9726dbaa4bcb9136c147d1d9ad42_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:03,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/3d1332116510470390f135160843add3, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:03,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/3d1332116510470390f135160843add3 is 175, key is test_row_0/A:col10/1731259501963/Put/seqid=0 2024-11-10T17:25:03,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742434_1610 (size=31255) 2024-11-10T17:25:03,746 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/3d1332116510470390f135160843add3 2024-11-10T17:25:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/632d0894675d4d1baf2ede03e113eee4 is 50, key is test_row_0/B:col10/1731259501963/Put/seqid=0 2024-11-10T17:25:03,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742435_1611 (size=12301) 2024-11-10T17:25:04,007 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-10T17:25:04,007 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-10T17:25:04,007 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,007 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,007 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,007 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,007 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,007 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-10T17:25:04,007 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,007 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,007 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,007 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,008 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-10T17:25:04,008 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,008 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,008 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,013 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-10T17:25:04,013 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,013 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,013 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T17:25:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36421 {}] regionserver.HRegion(8855): Flush requested on da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:04,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. as already flushing 2024-11-10T17:25:04,124 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-11-10T17:25:04,124 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,124 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,124 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,134 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-11-10T17:25:04,134 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-11-10T17:25:04,134 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,134 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,134 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,134 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,134 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,134 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,143 INFO [Thread-2392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2392. 2024-11-10T17:25:04,143 DEBUG [Thread-2392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:04,143 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,143 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:04,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:04,167 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/632d0894675d4d1baf2ede03e113eee4 2024-11-10T17:25:04,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/33183e35394e4df88fb6acf5e4d5ec99 is 50, key is test_row_0/C:col10/1731259501963/Put/seqid=0 2024-11-10T17:25:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742436_1612 (size=12301) 2024-11-10T17:25:04,581 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/33183e35394e4df88fb6acf5e4d5ec99 2024-11-10T17:25:04,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/3d1332116510470390f135160843add3 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/3d1332116510470390f135160843add3 2024-11-10T17:25:04,587 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/3d1332116510470390f135160843add3, entries=150, sequenceid=279, filesize=30.5 K 2024-11-10T17:25:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/632d0894675d4d1baf2ede03e113eee4 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/632d0894675d4d1baf2ede03e113eee4 2024-11-10T17:25:04,590 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/632d0894675d4d1baf2ede03e113eee4, entries=150, sequenceid=279, filesize=12.0 K 2024-11-10T17:25:04,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/33183e35394e4df88fb6acf5e4d5ec99 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33183e35394e4df88fb6acf5e4d5ec99 2024-11-10T17:25:04,593 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33183e35394e4df88fb6acf5e4d5ec99, entries=150, sequenceid=279, filesize=12.0 K 2024-11-10T17:25:04,593 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for da5a26d0da1375a506f6b0241d015eea in 1315ms, sequenceid=279, compaction requested=true 2024-11-10T17:25:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for da5a26d0da1375a506f6b0241d015eea: 2024-11-10T17:25:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-10T17:25:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-10T17:25:04,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-10T17:25:04,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-11-10T17:25:04,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.6350 sec 2024-11-10T17:25:04,874 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T17:25:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T17:25:05,112 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:25:06,163 INFO [Thread-2394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2394. 2024-11-10T17:25:06,163 DEBUG [Thread-2394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T17:25:06,163 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:06,164 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:06,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3665 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3563 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3721 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3666 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3578 2024-11-10T17:25:06,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T17:25:06,164 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:25:06,164 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:25:06,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:06,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:06,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:06,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T17:25:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T17:25:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T17:25:06,168 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259506168"}]},"ts":"1731259506168"} 2024-11-10T17:25:06,170 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T17:25:06,170 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T17:25:06,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T17:25:06,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, UNASSIGN}] 2024-11-10T17:25:06,172 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, UNASSIGN 2024-11-10T17:25:06,172 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=CLOSING, regionLocation=8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:06,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, UNASSIGN because future has completed 2024-11-10T17:25:06,174 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T17:25:06,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553}] 2024-11-10T17:25:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T17:25:06,326 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(122): Close da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:06,326 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T17:25:06,326 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1722): Closing da5a26d0da1375a506f6b0241d015eea, disabling compactions & flushes 2024-11-10T17:25:06,326 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:06,326 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:06,326 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. after waiting 0 ms 2024-11-10T17:25:06,326 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:06,326 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(2902): Flushing da5a26d0da1375a506f6b0241d015eea 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T17:25:06,332 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108b0a0f6541254fed9f535cf371bf5e87_da5a26d0da1375a506f6b0241d015eea is 50, key is test_row_0/A:col10/1731259504141/Put/seqid=0 2024-11-10T17:25:06,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742437_1613 (size=12454) 2024-11-10T17:25:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T17:25:06,736 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T17:25:06,740 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108b0a0f6541254fed9f535cf371bf5e87_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108b0a0f6541254fed9f535cf371bf5e87_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:06,740 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/149a34e0901c419786f7ca50498bd7b6, store: [table=TestAcidGuarantees family=A region=da5a26d0da1375a506f6b0241d015eea] 2024-11-10T17:25:06,741 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/149a34e0901c419786f7ca50498bd7b6 is 175, key is test_row_0/A:col10/1731259504141/Put/seqid=0 2024-11-10T17:25:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742438_1614 (size=31255) 2024-11-10T17:25:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T17:25:07,145 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/149a34e0901c419786f7ca50498bd7b6 2024-11-10T17:25:07,151 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/f2387fa54541446abb7e73c5cc47e82e is 50, key is test_row_0/B:col10/1731259504141/Put/seqid=0 2024-11-10T17:25:07,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742439_1615 (size=12301) 2024-11-10T17:25:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T17:25:07,555 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/f2387fa54541446abb7e73c5cc47e82e 2024-11-10T17:25:07,561 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/33f1625e7ac64a02bb1bd711e5d0d713 is 50, key is test_row_0/C:col10/1731259504141/Put/seqid=0 2024-11-10T17:25:07,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742440_1616 (size=12301) 2024-11-10T17:25:07,965 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/33f1625e7ac64a02bb1bd711e5d0d713 2024-11-10T17:25:07,968 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/A/149a34e0901c419786f7ca50498bd7b6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/149a34e0901c419786f7ca50498bd7b6 2024-11-10T17:25:07,972 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/149a34e0901c419786f7ca50498bd7b6, entries=150, sequenceid=287, filesize=30.5 K 2024-11-10T17:25:07,972 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/B/f2387fa54541446abb7e73c5cc47e82e as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2387fa54541446abb7e73c5cc47e82e 2024-11-10T17:25:07,975 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2387fa54541446abb7e73c5cc47e82e, entries=150, sequenceid=287, filesize=12.0 K 2024-11-10T17:25:07,975 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/.tmp/C/33f1625e7ac64a02bb1bd711e5d0d713 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33f1625e7ac64a02bb1bd711e5d0d713 2024-11-10T17:25:07,978 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33f1625e7ac64a02bb1bd711e5d0d713, entries=150, sequenceid=287, filesize=12.0 K 2024-11-10T17:25:07,978 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for da5a26d0da1375a506f6b0241d015eea in 1652ms, sequenceid=287, compaction requested=true 2024-11-10T17:25:07,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c5e7cb6c106f4d41bb8667a994fc87f6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/315bed0e0d7748098270fe997b92211c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/5ef026276dd44db3b52706990bd124c8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/771c70861f1a43e3b71b0bd488ffde1b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a0ef63225e1f4ac1ba0424c66cdfb417, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/cd80713c3c0e4fc6ac1f3b437819f2cb, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c35ad4793f0e4348a1197e759ca90815, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/4d4b4d896cd441d19cf81d70ac37942d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/8d6b927c4a5a40269c42158dce05cb7c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/48ca8c75bf1a415faf4312cbb64a040a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bd71cabb3b0a48f5b2b0023e0367e3b6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/10c750778ec84bbf9ed012523f6563bc, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bfbc7e527870459e9e3e1ec7595eaf8a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/75f9cb45f06b4ca9a1068d4777e4fbc5, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/013afd5e22644775b05604be748fc1b7, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/36db8ea2496a4c15b3e42da334b11d19] to archive 2024-11-10T17:25:07,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:25:07,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c5e7cb6c106f4d41bb8667a994fc87f6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c5e7cb6c106f4d41bb8667a994fc87f6 2024-11-10T17:25:07,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/315bed0e0d7748098270fe997b92211c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/315bed0e0d7748098270fe997b92211c 2024-11-10T17:25:07,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/5ef026276dd44db3b52706990bd124c8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/5ef026276dd44db3b52706990bd124c8 2024-11-10T17:25:07,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/771c70861f1a43e3b71b0bd488ffde1b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/771c70861f1a43e3b71b0bd488ffde1b 2024-11-10T17:25:07,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a0ef63225e1f4ac1ba0424c66cdfb417 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a0ef63225e1f4ac1ba0424c66cdfb417 2024-11-10T17:25:07,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/cd80713c3c0e4fc6ac1f3b437819f2cb to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/cd80713c3c0e4fc6ac1f3b437819f2cb 2024-11-10T17:25:07,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c35ad4793f0e4348a1197e759ca90815 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/c35ad4793f0e4348a1197e759ca90815 2024-11-10T17:25:07,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/4d4b4d896cd441d19cf81d70ac37942d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/4d4b4d896cd441d19cf81d70ac37942d 2024-11-10T17:25:07,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/8d6b927c4a5a40269c42158dce05cb7c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/8d6b927c4a5a40269c42158dce05cb7c 2024-11-10T17:25:07,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/48ca8c75bf1a415faf4312cbb64a040a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/48ca8c75bf1a415faf4312cbb64a040a 2024-11-10T17:25:07,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bd71cabb3b0a48f5b2b0023e0367e3b6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bd71cabb3b0a48f5b2b0023e0367e3b6 2024-11-10T17:25:07,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/10c750778ec84bbf9ed012523f6563bc to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/10c750778ec84bbf9ed012523f6563bc 2024-11-10T17:25:07,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bfbc7e527870459e9e3e1ec7595eaf8a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/bfbc7e527870459e9e3e1ec7595eaf8a 2024-11-10T17:25:07,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/75f9cb45f06b4ca9a1068d4777e4fbc5 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/75f9cb45f06b4ca9a1068d4777e4fbc5 2024-11-10T17:25:07,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/013afd5e22644775b05604be748fc1b7 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/013afd5e22644775b05604be748fc1b7 2024-11-10T17:25:07,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/36db8ea2496a4c15b3e42da334b11d19 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/36db8ea2496a4c15b3e42da334b11d19 2024-11-10T17:25:07,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/dd4c62422bfa4765a7abb88d79dafca1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/79ab6c53870a4dd0a96457a7418be859, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/6392cf94344944d9a70523ae67d9aaa3, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/82dd721f04b94c0886ad4b73cf424aa8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e9fe16c4ebc8477b90d5fa44a65737f8, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e63a84241d894a53be9ca2e09613dfab, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/c91fe3cf7dad4f50bfe75e2a39fb5c33, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/561b166b742b4fb795748b8f33d59791, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/74b83399b0ec494bbddc90ebaac75116, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/1bc875bcc517411e901dd1770169c6e2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/5547a90e77a34d7e836499c4ad79bf3e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/745c0839fb7a4c82973ee063e588cd05, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/eede62149b0c432a8fe3065b8482c337, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/849bb094cbae42199ce8c3e54e4b237a, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2472d8ce0524018a0870d8676f6cf6f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e2177a514de34c1a9dfcfb696d09c036] to archive 2024-11-10T17:25:07,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:25:07,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/dd4c62422bfa4765a7abb88d79dafca1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/dd4c62422bfa4765a7abb88d79dafca1 2024-11-10T17:25:07,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/79ab6c53870a4dd0a96457a7418be859 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/79ab6c53870a4dd0a96457a7418be859 2024-11-10T17:25:07,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/6392cf94344944d9a70523ae67d9aaa3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/6392cf94344944d9a70523ae67d9aaa3 2024-11-10T17:25:07,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/82dd721f04b94c0886ad4b73cf424aa8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/82dd721f04b94c0886ad4b73cf424aa8 2024-11-10T17:25:07,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e9fe16c4ebc8477b90d5fa44a65737f8 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e9fe16c4ebc8477b90d5fa44a65737f8 2024-11-10T17:25:07,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e63a84241d894a53be9ca2e09613dfab to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e63a84241d894a53be9ca2e09613dfab 2024-11-10T17:25:08,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/c91fe3cf7dad4f50bfe75e2a39fb5c33 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/c91fe3cf7dad4f50bfe75e2a39fb5c33 2024-11-10T17:25:08,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/561b166b742b4fb795748b8f33d59791 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/561b166b742b4fb795748b8f33d59791 2024-11-10T17:25:08,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/74b83399b0ec494bbddc90ebaac75116 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/74b83399b0ec494bbddc90ebaac75116 2024-11-10T17:25:08,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/1bc875bcc517411e901dd1770169c6e2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/1bc875bcc517411e901dd1770169c6e2 2024-11-10T17:25:08,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/5547a90e77a34d7e836499c4ad79bf3e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/5547a90e77a34d7e836499c4ad79bf3e 2024-11-10T17:25:08,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/745c0839fb7a4c82973ee063e588cd05 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/745c0839fb7a4c82973ee063e588cd05 2024-11-10T17:25:08,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/eede62149b0c432a8fe3065b8482c337 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/eede62149b0c432a8fe3065b8482c337 2024-11-10T17:25:08,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/849bb094cbae42199ce8c3e54e4b237a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/849bb094cbae42199ce8c3e54e4b237a 2024-11-10T17:25:08,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2472d8ce0524018a0870d8676f6cf6f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2472d8ce0524018a0870d8676f6cf6f 2024-11-10T17:25:08,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e2177a514de34c1a9dfcfb696d09c036 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/e2177a514de34c1a9dfcfb696d09c036 2024-11-10T17:25:08,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b3c29464ff584fbe8afdd7ab8c1a0447, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/13635f524b4c4094b6bcc8e196f82e1d, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/6954eb7cb3ad4d01be59b0046ccdb6a1, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/58cb5f1e9d9c46bea3667a92719e362c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/d0096e34321d44078537c041b0231e1b, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/85e19ee45e354340be7152e2d12e878f, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/cecb1a6ba50448cbbef49a638b8b38f6, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/c3072f94557446cba73c893a5f15485c, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/f38b77d471714ea09e502258025b49b2, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7106c049d95a4bc1a8b19b282656fd92, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b7b8fa95064942a7a29ded32a35ee9df, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/a2a0f7ba7ea94c2da220075cc8d7ce09, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7d0cc5a5142040b8a347e5605cba7c85, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/2812d4c48a2b4367bb0a9d6778f6984e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/984a9759ae8b40639d0079ae1f851f9e, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/1ae1295002764ed89b70ddbfa7356a04] to archive 2024-11-10T17:25:08,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T17:25:08,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b3c29464ff584fbe8afdd7ab8c1a0447 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b3c29464ff584fbe8afdd7ab8c1a0447 2024-11-10T17:25:08,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/13635f524b4c4094b6bcc8e196f82e1d to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/13635f524b4c4094b6bcc8e196f82e1d 2024-11-10T17:25:08,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/6954eb7cb3ad4d01be59b0046ccdb6a1 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/6954eb7cb3ad4d01be59b0046ccdb6a1 2024-11-10T17:25:08,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/58cb5f1e9d9c46bea3667a92719e362c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/58cb5f1e9d9c46bea3667a92719e362c 2024-11-10T17:25:08,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/d0096e34321d44078537c041b0231e1b to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/d0096e34321d44078537c041b0231e1b 2024-11-10T17:25:08,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/85e19ee45e354340be7152e2d12e878f to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/85e19ee45e354340be7152e2d12e878f 2024-11-10T17:25:08,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/cecb1a6ba50448cbbef49a638b8b38f6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/cecb1a6ba50448cbbef49a638b8b38f6 2024-11-10T17:25:08,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/c3072f94557446cba73c893a5f15485c to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/c3072f94557446cba73c893a5f15485c 2024-11-10T17:25:08,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/f38b77d471714ea09e502258025b49b2 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/f38b77d471714ea09e502258025b49b2 2024-11-10T17:25:08,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7106c049d95a4bc1a8b19b282656fd92 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7106c049d95a4bc1a8b19b282656fd92 2024-11-10T17:25:08,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b7b8fa95064942a7a29ded32a35ee9df to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/b7b8fa95064942a7a29ded32a35ee9df 2024-11-10T17:25:08,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/a2a0f7ba7ea94c2da220075cc8d7ce09 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/a2a0f7ba7ea94c2da220075cc8d7ce09 2024-11-10T17:25:08,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7d0cc5a5142040b8a347e5605cba7c85 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/7d0cc5a5142040b8a347e5605cba7c85 2024-11-10T17:25:08,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/2812d4c48a2b4367bb0a9d6778f6984e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/2812d4c48a2b4367bb0a9d6778f6984e 2024-11-10T17:25:08,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/984a9759ae8b40639d0079ae1f851f9e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/984a9759ae8b40639d0079ae1f851f9e 2024-11-10T17:25:08,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/1ae1295002764ed89b70ddbfa7356a04 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/1ae1295002764ed89b70ddbfa7356a04 2024-11-10T17:25:08,029 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/recovered.edits/290.seqid, newMaxSeqId=290, maxSeqId=4 2024-11-10T17:25:08,029 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. 2024-11-10T17:25:08,029 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1676): Region close journal for da5a26d0da1375a506f6b0241d015eea: Waiting for close lock at 1731259506326Running coprocessor pre-close hooks at 1731259506326Disabling compacts and flushes for region at 1731259506326Disabling writes for close at 1731259506326Obtaining lock to block concurrent updates at 1731259506326Preparing flush snapshotting stores in da5a26d0da1375a506f6b0241d015eea at 1731259506326Finished memstore snapshotting TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731259506327 (+1 ms)Flushing stores of TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea. at 1731259506327Flushing da5a26d0da1375a506f6b0241d015eea/A: creating writer at 1731259506327Flushing mob file da5a26d0da1375a506f6b0241d015eea/A: appending metadata at 1731259506332 (+5 ms)Flushing mob file da5a26d0da1375a506f6b0241d015eea/A: closing flushed file at 1731259506332Flushing da5a26d0da1375a506f6b0241d015eea/A: appending metadata at 1731259506740 (+408 ms)Flushing da5a26d0da1375a506f6b0241d015eea/A: closing flushed file at 1731259506741 (+1 ms)Flushing da5a26d0da1375a506f6b0241d015eea/B: creating writer at 1731259507148 (+407 ms)Flushing da5a26d0da1375a506f6b0241d015eea/B: appending metadata at 1731259507150 (+2 ms)Flushing da5a26d0da1375a506f6b0241d015eea/B: closing flushed file at 1731259507150Flushing da5a26d0da1375a506f6b0241d015eea/C: creating writer at 1731259507558 (+408 ms)Flushing da5a26d0da1375a506f6b0241d015eea/C: appending metadata at 1731259507561 (+3 ms)Flushing da5a26d0da1375a506f6b0241d015eea/C: closing flushed file at 1731259507561Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31664522: reopening flushed file at 1731259507968 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1924ff1a: reopening flushed file at 1731259507972 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44fad4e7: reopening flushed file at 1731259507975 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for da5a26d0da1375a506f6b0241d015eea in 1652ms, sequenceid=287, compaction requested=true at 1731259507978 (+3 ms)Writing region close event to WAL at 1731259508026 (+48 ms)Running coprocessor post-close hooks at 1731259508029 (+3 ms)Closed at 1731259508029 2024-11-10T17:25:08,031 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(157): Closed da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,031 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=da5a26d0da1375a506f6b0241d015eea, regionState=CLOSED 2024-11-10T17:25:08,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 because future has completed 2024-11-10T17:25:08,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T17:25:08,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseRegionProcedure da5a26d0da1375a506f6b0241d015eea, server=8e3de9160f9f,36421,1731259326553 in 1.8590 sec 2024-11-10T17:25:08,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-10T17:25:08,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da5a26d0da1375a506f6b0241d015eea, UNASSIGN in 1.8630 sec 2024-11-10T17:25:08,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T17:25:08,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-10T17:25:08,038 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731259508038"}]},"ts":"1731259508038"} 2024-11-10T17:25:08,039 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T17:25:08,039 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T17:25:08,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8750 sec 2024-11-10T17:25:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T17:25:08,311 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:25:08,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T17:25:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:25:08,313 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:25:08,314 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=156, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:25:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T17:25:08,316 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,317 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C, FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/recovered.edits] 2024-11-10T17:25:08,319 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/149a34e0901c419786f7ca50498bd7b6 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/149a34e0901c419786f7ca50498bd7b6 2024-11-10T17:25:08,320 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/3d1332116510470390f135160843add3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/3d1332116510470390f135160843add3 2024-11-10T17:25:08,321 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/3fc8e205666e440c9ae5222d7130fd26 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/3fc8e205666e440c9ae5222d7130fd26 2024-11-10T17:25:08,322 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a29d330905d343eb9f7cbe6b31f27df3 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/A/a29d330905d343eb9f7cbe6b31f27df3 2024-11-10T17:25:08,323 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/632d0894675d4d1baf2ede03e113eee4 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/632d0894675d4d1baf2ede03e113eee4 2024-11-10T17:25:08,324 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/a97c0e0e7bf640db9fb15c0edcd9f609 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/a97c0e0e7bf640db9fb15c0edcd9f609 2024-11-10T17:25:08,325 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/ec5f34cdd1bf45bea48ff9aada36be0a to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/ec5f34cdd1bf45bea48ff9aada36be0a 2024-11-10T17:25:08,326 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2387fa54541446abb7e73c5cc47e82e to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/B/f2387fa54541446abb7e73c5cc47e82e 2024-11-10T17:25:08,328 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/250c6878092c43ab934cbba6c77ab942 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/250c6878092c43ab934cbba6c77ab942 2024-11-10T17:25:08,329 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33183e35394e4df88fb6acf5e4d5ec99 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33183e35394e4df88fb6acf5e4d5ec99 2024-11-10T17:25:08,330 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33f1625e7ac64a02bb1bd711e5d0d713 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/33f1625e7ac64a02bb1bd711e5d0d713 2024-11-10T17:25:08,330 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/e2d8a3d7dfc443559d66062c32118992 to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/C/e2d8a3d7dfc443559d66062c32118992 2024-11-10T17:25:08,333 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/recovered.edits/290.seqid to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea/recovered.edits/290.seqid 2024-11-10T17:25:08,333 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/default/TestAcidGuarantees/da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,333 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T17:25:08,334 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T17:25:08,334 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T17:25:08,336 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e4ee129954f4ae391874e0ee62d29cd_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e4ee129954f4ae391874e0ee62d29cd_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,337 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103768c29d6d0b422f95e366b32516ac4c_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103768c29d6d0b422f95e366b32516ac4c_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,338 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111040d50bd79a3448a38bb9e407036e02f9_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111040d50bd79a3448a38bb9e407036e02f9_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,339 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110556f5c7c12e24f5eb6acbaa4b859047b_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110556f5c7c12e24f5eb6acbaa4b859047b_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,340 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058f7cbf6100b47ee95e8039e98d8438d_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058f7cbf6100b47ee95e8039e98d8438d_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,341 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a94f51f02e54e7cba756e9f98210948_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a94f51f02e54e7cba756e9f98210948_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,342 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107aca3a51d55b4698b4afc579aaf3628c_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107aca3a51d55b4698b4afc579aaf3628c_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,343 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b719cd053b84318a0b4d99e551afc8b_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107b719cd053b84318a0b4d99e551afc8b_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,344 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108b0a0f6541254fed9f535cf371bf5e87_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108b0a0f6541254fed9f535cf371bf5e87_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,344 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a5ee9726dbaa4bcb9136c147d1d9ad42_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a5ee9726dbaa4bcb9136c147d1d9ad42_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,345 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a8d401c8f3824010afd3d67ef6eabe4a_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a8d401c8f3824010afd3d67ef6eabe4a_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,346 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1e403624a89477a86195dd109c309a1_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b1e403624a89477a86195dd109c309a1_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,347 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bf45ae110a0b4041b5065197f23b3ffb_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bf45ae110a0b4041b5065197f23b3ffb_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,348 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d60f6bd507e64ac9a8c664b958bf914c_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d60f6bd507e64ac9a8c664b958bf914c_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,349 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e285d7e0af72475684f15d5f19126947_da5a26d0da1375a506f6b0241d015eea to hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e285d7e0af72475684f15d5f19126947_da5a26d0da1375a506f6b0241d015eea 2024-11-10T17:25:08,350 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T17:25:08,352 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=156, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:25:08,354 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T17:25:08,356 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T17:25:08,357 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=156, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:25:08,357 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T17:25:08,357 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731259508357"}]},"ts":"9223372036854775807"} 2024-11-10T17:25:08,359 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T17:25:08,359 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => da5a26d0da1375a506f6b0241d015eea, NAME => 'TestAcidGuarantees,,1731259480899.da5a26d0da1375a506f6b0241d015eea.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T17:25:08,359 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T17:25:08,359 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731259508359"}]},"ts":"9223372036854775807"} 2024-11-10T17:25:08,360 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T17:25:08,361 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=156, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T17:25:08,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-10T17:25:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40007 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T17:25:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T17:25:08,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T17:25:08,431 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 236) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=337 (was 370), ProcessCount=11 (was 11), AvailableMemoryMB=5759 (was 5859) 2024-11-10T17:25:08,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T17:25:08,432 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:25:08,432 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:25:08,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:08,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:08,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T17:25:08,432 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T17:25:08,432 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1143773905, stopped=false 2024-11-10T17:25:08,433 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8e3de9160f9f,40007,1731259325831 2024-11-10T17:25:08,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T17:25:08,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T17:25:08,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:25:08,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:25:08,435 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T17:25:08,435 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T17:25:08,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T17:25:08,435 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:25:08,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:08,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T17:25:08,436 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8e3de9160f9f,36421,1731259326553' ***** 2024-11-10T17:25:08,436 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T17:25:08,436 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(959): stopping server 8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8e3de9160f9f:36421. 2024-11-10T17:25:08,437 DEBUG [RS:0;8e3de9160f9f:36421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T17:25:08,437 DEBUG [RS:0;8e3de9160f9f:36421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T17:25:08,437 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T17:25:08,438 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T17:25:08,438 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T17:25:08,438 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T17:25:08,439 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T17:25:08,439 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T17:25:08,439 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T17:25:08,439 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T17:25:08,439 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T17:25:08,439 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T17:25:08,439 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T17:25:08,462 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/info/cf3d562c63c14f3e845ac303e0f39ba6 is 95, key is TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49./info:/1731259355560/DeleteFamily/seqid=0 2024-11-10T17:25:08,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742441_1617 (size=6514) 2024-11-10T17:25:08,488 INFO [regionserver/8e3de9160f9f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T17:25:08,639 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T17:25:08,839 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T17:25:08,866 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/info/cf3d562c63c14f3e845ac303e0f39ba6 2024-11-10T17:25:08,884 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/ns/044e228842684d6c8328775216101f39 is 93, key is TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49./ns:/1731259355507/DeleteFamily/seqid=0 2024-11-10T17:25:08,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742442_1618 (size=5906) 2024-11-10T17:25:09,039 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T17:25:09,239 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T17:25:09,285 INFO [regionserver/8e3de9160f9f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T17:25:09,285 INFO [regionserver/8e3de9160f9f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T17:25:09,288 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/ns/044e228842684d6c8328775216101f39 2024-11-10T17:25:09,305 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/rep_barrier/eb8c82a8c41548218723d5446928d935 is 102, key is TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49./rep_barrier:/1731259355507/DeleteFamily/seqid=0 2024-11-10T17:25:09,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742443_1619 (size=6025) 2024-11-10T17:25:09,440 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T17:25:09,640 DEBUG [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T17:25:09,710 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/rep_barrier/eb8c82a8c41548218723d5446928d935 2024-11-10T17:25:09,729 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/table/a44e4d97f1b546039c5d120e6f38ebfe is 96, key is TestAcidGuarantees,,1731259329780.86036b2e5b1a6747cefd43fa6b653d49./table:/1731259355507/DeleteFamily/seqid=0 2024-11-10T17:25:09,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742444_1620 (size=5936) 2024-11-10T17:25:09,732 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/table/a44e4d97f1b546039c5d120e6f38ebfe 2024-11-10T17:25:09,736 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/info/cf3d562c63c14f3e845ac303e0f39ba6 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/info/cf3d562c63c14f3e845ac303e0f39ba6 2024-11-10T17:25:09,738 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/info/cf3d562c63c14f3e845ac303e0f39ba6, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T17:25:09,739 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/ns/044e228842684d6c8328775216101f39 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/ns/044e228842684d6c8328775216101f39 2024-11-10T17:25:09,741 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/ns/044e228842684d6c8328775216101f39, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T17:25:09,742 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/rep_barrier/eb8c82a8c41548218723d5446928d935 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/rep_barrier/eb8c82a8c41548218723d5446928d935 2024-11-10T17:25:09,745 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/rep_barrier/eb8c82a8c41548218723d5446928d935, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T17:25:09,745 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/.tmp/table/a44e4d97f1b546039c5d120e6f38ebfe as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/table/a44e4d97f1b546039c5d120e6f38ebfe 2024-11-10T17:25:09,748 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/table/a44e4d97f1b546039c5d120e6f38ebfe, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T17:25:09,748 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1309ms, sequenceid=90, compaction requested=false 2024-11-10T17:25:09,754 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T17:25:09,754 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T17:25:09,754 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T17:25:09,755 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731259508439Running coprocessor pre-close hooks at 1731259508439Disabling compacts and flushes for region at 1731259508439Disabling writes for close at 1731259508439Obtaining lock to block concurrent updates at 1731259508439Preparing flush snapshotting stores in 1588230740 at 1731259508439Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731259508439Flushing stores of hbase:meta,,1.1588230740 at 1731259508440 (+1 ms)Flushing 1588230740/info: creating writer at 1731259508441 (+1 ms)Flushing 1588230740/info: appending metadata at 1731259508462 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731259508462Flushing 1588230740/ns: creating writer at 1731259508870 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731259508884 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731259508884Flushing 1588230740/rep_barrier: creating writer at 1731259509291 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731259509305 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731259509305Flushing 1588230740/table: creating writer at 1731259509713 (+408 ms)Flushing 1588230740/table: appending metadata at 1731259509729 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731259509729Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a490015: reopening flushed file at 1731259509735 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63a61ca0: reopening flushed file at 1731259509738 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18dfc364: reopening flushed file at 1731259509741 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ae29b4e: reopening flushed file at 1731259509745 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1309ms, sequenceid=90, compaction requested=false at 1731259509748 (+3 ms)Writing region close event to WAL at 1731259509751 (+3 ms)Running coprocessor post-close hooks at 1731259509754 (+3 ms)Closed at 1731259509754 2024-11-10T17:25:09,755 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T17:25:09,840 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(976): stopping server 8e3de9160f9f,36421,1731259326553; all regions closed. 2024-11-10T17:25:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741834_1010 (size=25344) 2024-11-10T17:25:09,846 DEBUG [RS:0;8e3de9160f9f:36421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/oldWALs 2024-11-10T17:25:09,846 INFO [RS:0;8e3de9160f9f:36421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8e3de9160f9f%2C36421%2C1731259326553.meta:.meta(num 1731259329354) 2024-11-10T17:25:09,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741832_1008 (size=14197614) 2024-11-10T17:25:09,849 DEBUG [RS:0;8e3de9160f9f:36421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/oldWALs 2024-11-10T17:25:09,850 INFO [RS:0;8e3de9160f9f:36421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8e3de9160f9f%2C36421%2C1731259326553:(num 1731259328449) 2024-11-10T17:25:09,850 DEBUG [RS:0;8e3de9160f9f:36421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T17:25:09,850 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T17:25:09,850 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T17:25:09,850 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.ChoreService(370): Chore service for: regionserver/8e3de9160f9f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T17:25:09,850 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T17:25:09,851 INFO [RS:0;8e3de9160f9f:36421 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36421 2024-11-10T17:25:09,850 INFO [regionserver/8e3de9160f9f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T17:25:09,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8e3de9160f9f,36421,1731259326553 2024-11-10T17:25:09,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T17:25:09,856 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T17:25:09,858 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8e3de9160f9f,36421,1731259326553] 2024-11-10T17:25:09,859 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8e3de9160f9f,36421,1731259326553 already deleted, retry=false 2024-11-10T17:25:09,859 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8e3de9160f9f,36421,1731259326553 expired; onlineServers=0 2024-11-10T17:25:09,860 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8e3de9160f9f,40007,1731259325831' ***** 2024-11-10T17:25:09,860 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T17:25:09,860 INFO [M:0;8e3de9160f9f:40007 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T17:25:09,860 INFO [M:0;8e3de9160f9f:40007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T17:25:09,860 DEBUG [M:0;8e3de9160f9f:40007 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T17:25:09,860 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T17:25:09,860 DEBUG [M:0;8e3de9160f9f:40007 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T17:25:09,860 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.large.0-1731259328090 {}] cleaner.HFileCleaner(306): Exit Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.large.0-1731259328090,5,FailOnTimeoutGroup] 2024-11-10T17:25:09,860 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.small.0-1731259328096 {}] cleaner.HFileCleaner(306): Exit Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.small.0-1731259328096,5,FailOnTimeoutGroup] 2024-11-10T17:25:09,860 INFO [M:0;8e3de9160f9f:40007 {}] hbase.ChoreService(370): Chore service for: master/8e3de9160f9f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T17:25:09,860 INFO [M:0;8e3de9160f9f:40007 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T17:25:09,861 DEBUG [M:0;8e3de9160f9f:40007 {}] master.HMaster(1795): Stopping service threads 2024-11-10T17:25:09,861 INFO [M:0;8e3de9160f9f:40007 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T17:25:09,861 INFO [M:0;8e3de9160f9f:40007 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T17:25:09,861 ERROR [M:0;8e3de9160f9f:40007 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T17:25:09,862 INFO [M:0;8e3de9160f9f:40007 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T17:25:09,862 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T17:25:09,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T17:25:09,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T17:25:09,862 DEBUG [M:0;8e3de9160f9f:40007 {}] zookeeper.ZKUtil(347): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T17:25:09,862 WARN [M:0;8e3de9160f9f:40007 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T17:25:09,863 INFO [M:0;8e3de9160f9f:40007 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/.lastflushedseqids 2024-11-10T17:25:09,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742445_1621 (size=72) 2024-11-10T17:25:09,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T17:25:09,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36421-0x10102e6c4a10001, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T17:25:09,959 INFO [RS:0;8e3de9160f9f:36421 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T17:25:09,959 INFO [RS:0;8e3de9160f9f:36421 {}] regionserver.HRegionServer(1031): Exiting; stopping=8e3de9160f9f,36421,1731259326553; zookeeper connection closed. 2024-11-10T17:25:09,959 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@30b6df19 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@30b6df19 2024-11-10T17:25:09,960 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T17:25:10,275 INFO [M:0;8e3de9160f9f:40007 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T17:25:10,275 INFO [M:0;8e3de9160f9f:40007 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T17:25:10,275 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T17:25:10,275 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:25:10,275 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:25:10,275 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T17:25:10,275 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:25:10,275 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=721.99 KB heapSize=885.76 KB 2024-11-10T17:25:10,291 DEBUG [M:0;8e3de9160f9f:40007 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bf55e318bdf4581ba57ca57298b3840 is 82, key is hbase:meta,,1/info:regioninfo/1731259329437/Put/seqid=0 2024-11-10T17:25:10,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742446_1622 (size=5672) 2024-11-10T17:25:10,698 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2021 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bf55e318bdf4581ba57ca57298b3840 2024-11-10T17:25:10,722 DEBUG [M:0;8e3de9160f9f:40007 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00e32b5ac0364ccd9c145627abba0b76 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x85/proc:d/1731259483975/Put/seqid=0 2024-11-10T17:25:10,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742447_1623 (size=41689) 2024-11-10T17:25:11,130 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721.44 KB at sequenceid=2021 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00e32b5ac0364ccd9c145627abba0b76 2024-11-10T17:25:11,133 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00e32b5ac0364ccd9c145627abba0b76 2024-11-10T17:25:11,147 DEBUG [M:0;8e3de9160f9f:40007 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6469e9bb8a34fcab38655925fd43bbf is 69, key is 8e3de9160f9f,36421,1731259326553/rs:state/1731259328204/Put/seqid=0 2024-11-10T17:25:11,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073742448_1624 (size=5156) 2024-11-10T17:25:11,151 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2021 (bloomFilter=true), to=hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6469e9bb8a34fcab38655925fd43bbf 2024-11-10T17:25:11,153 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bf55e318bdf4581ba57ca57298b3840 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7bf55e318bdf4581ba57ca57298b3840 2024-11-10T17:25:11,156 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7bf55e318bdf4581ba57ca57298b3840, entries=8, sequenceid=2021, filesize=5.5 K 2024-11-10T17:25:11,157 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00e32b5ac0364ccd9c145627abba0b76 as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00e32b5ac0364ccd9c145627abba0b76 2024-11-10T17:25:11,159 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00e32b5ac0364ccd9c145627abba0b76 2024-11-10T17:25:11,159 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00e32b5ac0364ccd9c145627abba0b76, entries=156, sequenceid=2021, filesize=40.7 K 2024-11-10T17:25:11,160 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6469e9bb8a34fcab38655925fd43bbf as hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6469e9bb8a34fcab38655925fd43bbf 2024-11-10T17:25:11,163 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37703/user/jenkins/test-data/775a1ca7-9d4f-feb9-b42a-61e219eff46a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6469e9bb8a34fcab38655925fd43bbf, entries=1, sequenceid=2021, filesize=5.0 K 2024-11-10T17:25:11,163 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(3140): Finished flush of dataSize ~721.99 KB/739320, heapSize ~885.46 KB/906712, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 888ms, sequenceid=2021, compaction requested=false 2024-11-10T17:25:11,175 INFO [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T17:25:11,175 DEBUG [M:0;8e3de9160f9f:40007 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731259510275Disabling compacts and flushes for region at 1731259510275Disabling writes for close at 1731259510275Obtaining lock to block concurrent updates at 1731259510275Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731259510275Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=739320, getHeapSize=906952, getOffHeapSize=0, getCellsCount=2202 at 1731259510276 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731259510277 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731259510277Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731259510291 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731259510291Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731259510702 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731259510721 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731259510721Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731259511133 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731259511147 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731259511147Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20232edc: reopening flushed file at 1731259511153 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b10e73c: reopening flushed file at 1731259511156 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42e49a9f: reopening flushed file at 1731259511159 (+3 ms)Finished flush of dataSize ~721.99 KB/739320, heapSize ~885.46 KB/906712, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 888ms, sequenceid=2021, compaction requested=false at 1731259511163 (+4 ms)Writing region close event to WAL at 1731259511174 (+11 ms)Closed at 1731259511175 (+1 ms) 2024-11-10T17:25:11,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36153 is added to blk_1073741830_1006 (size=871828) 2024-11-10T17:25:11,178 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T17:25:11,178 INFO [M:0;8e3de9160f9f:40007 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T17:25:11,178 INFO [M:0;8e3de9160f9f:40007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40007 2024-11-10T17:25:11,178 INFO [M:0;8e3de9160f9f:40007 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T17:25:11,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T17:25:11,282 INFO [M:0;8e3de9160f9f:40007 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T17:25:11,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40007-0x10102e6c4a10000, quorum=127.0.0.1:49964, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T17:25:11,290 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T17:25:11,292 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T17:25:11,292 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T17:25:11,292 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T17:25:11,292 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/hadoop.log.dir/,STOPPED} 2024-11-10T17:25:11,295 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T17:25:11,295 WARN [BP-838508068-172.17.0.2-1731259322633 heartbeating to localhost/127.0.0.1:37703 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T17:25:11,295 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T17:25:11,295 WARN [BP-838508068-172.17.0.2-1731259322633 heartbeating to localhost/127.0.0.1:37703 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-838508068-172.17.0.2-1731259322633 (Datanode Uuid 4ee7403d-08d9-4cb1-a51e-17e70bcbf226) service to localhost/127.0.0.1:37703 2024-11-10T17:25:11,297 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/data/data1/current/BP-838508068-172.17.0.2-1731259322633 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T17:25:11,298 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/cluster_97019d23-b652-ea62-87e9-99abace081b9/data/data2/current/BP-838508068-172.17.0.2-1731259322633 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T17:25:11,298 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T17:25:11,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T17:25:11,307 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T17:25:11,307 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T17:25:11,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T17:25:11,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9388d05-296a-05e2-c269-2e4fe0de63ce/hadoop.log.dir/,STOPPED} 2024-11-10T17:25:11,323 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T17:25:11,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down